Merge branch 'master' into feature/sql

Original commit: elastic/x-pack-elasticsearch@67f8321368

docs/build.gradle

@@ -9,7 +9,7 @@ apply plugin: 'elasticsearch.docs-test'
  * only remove entries from this list. When it is empty we'll remove it
  * entirely and have a party! There will be cake and everything.... */
 buildRestTests.expectedUnconvertedCandidates = [
-        'en/ml/getting-started.asciidoc',
+        'en/ml/getting-started-data.asciidoc',
         'en/ml/functions/count.asciidoc',
         'en/ml/functions/geo.asciidoc',
         'en/ml/functions/info.asciidoc',

docs/en/ml/getting-started-data.asciidoc

@@ -0,0 +1,219 @@
+[[ml-gs-data]]
+=== Identifying Data for Analysis
+
+For the purposes of this tutorial, we provide sample data that you can play with
+and search in {es}. When you consider your own data, however, it's important to
+take a moment and think about where the {xpackml} features will be most
+impactful.
+
+The first consideration is that it must be time series data. The {ml} features
+are designed to model and detect anomalies in time series data.
+
+The second consideration, especially when you are first learning to use {ml},
+is the importance of the data and how familiar you are with it. Ideally, it is
+information that contains key performance indicators (KPIs) for the health,
+security, or success of your business or system. It is information that you need
+to monitor and act on when anomalous behavior occurs. You might even have {kib}
+dashboards that you're already using to watch this data. The better you know the
+data, the quicker you will be able to create {ml} jobs that generate useful
+insights.
+
+The final consideration is where the data is located. This tutorial assumes that
+your data is stored in {es}. It guides you through the steps required to create
+a _{dfeed}_ that passes data to a job. If your own data is outside of {es},
+analysis is still possible by using a post data API.
+
+IMPORTANT: If you want to create {ml} jobs in {kib}, you must use {dfeeds}.
+That is to say, you must store your input data in {es}. When you create
+a job, you select an existing index pattern and {kib} configures the {dfeed}
+for you under the covers.
+
+
+[float]
+[[ml-gs-sampledata]]
+==== Obtaining a Sample Data Set
+
+In this step we will upload some sample data to {es}. This is standard
+{es} functionality, and is needed to set the stage for using {ml}.
+
+The sample data for this tutorial contains information about the requests that
+are received by various applications and services in a system. A system
+administrator might use this type of information to track the total number of
+requests across all of the infrastructure. If the number of requests increases
+or decreases unexpectedly, for example, this might be an indication that there
+is a problem or that resources need to be redistributed. By using the {xpack}
+{ml} features to model the behavior of this data, it is easier to identify
+anomalies and take appropriate action.
+
+Download this sample data by clicking here:
+https://download.elastic.co/demos/machine_learning/gettingstarted/server_metrics.tar.gz[server_metrics.tar.gz]
+
+Use the following commands to extract the files:
+
+[source,shell]
+----------------------------------
+tar -zxvf server_metrics.tar.gz
+----------------------------------
+
+Each document in the server-metrics data set has the following schema:
+
+[source,js]
+----------------------------------
+{
+  "index":
+  {
+    "_index":"server-metrics",
+    "_type":"metric",
+    "_id":"1177"
+  }
+}
+{
+  "@timestamp":"2017-03-23T13:00:00",
+  "accept":36320,
+  "deny":4156,
+  "host":"server_2",
+  "response":2.4558210155,
+  "service":"app_3",
+  "total":40476
+}
+----------------------------------
+
+TIP: The sample data sets include summarized data. For example, the `total`
+value is a sum of the requests that were received by a specific service at a
+particular time. If your data is stored in {es}, you can generate
+this type of sum or average by using aggregations. One of the benefits of
+summarizing data this way is that {es} automatically distributes
+these calculations across your cluster. You can then feed this summarized data
+into {xpackml} instead of raw results, which reduces the volume
+of data that must be considered while detecting anomalies. For the purposes of
+this tutorial, however, these summary values are stored in {es}. For more
+information, see <<ml-configuring-aggregation>>.
+
+Before you load the data set, you need to set up {ref}/mapping.html[_mappings_]
+for the fields. Mappings divide the documents in the index into logical groups
+and specify a field's characteristics, such as the field's searchability or
+whether or not it's _tokenized_, or broken up into separate words.
+
+The sample data includes an `upload_server-metrics.sh` script, which you can use
+to create the mappings and load the data set. You can download it by clicking
+here: https://download.elastic.co/demos/machine_learning/gettingstarted/upload_server-metrics.sh[upload_server-metrics.sh]
+Before you run it, however, you must edit the USERNAME and PASSWORD variables
+with your actual user ID and password.
+
+The script runs a command similar to the following example, which sets up a
+mapping for the data set:
+
+[source,shell]
+----------------------------------
+
+curl -u elastic:x-pack-test-password -X PUT -H 'Content-Type: application/json'
+http://localhost:9200/server-metrics -d '{
+   "settings":{
+      "number_of_shards":1,
+      "number_of_replicas":0
+   },
+   "mappings":{
+      "metric":{
+         "properties":{
+            "@timestamp":{
+               "type":"date"
+            },
+            "accept":{
+               "type":"long"
+            },
+            "deny":{
+               "type":"long"
+            },
+            "host":{
+               "type":"keyword"
+            },
+            "response":{
+               "type":"float"
+            },
+            "service":{
+               "type":"keyword"
+            },
+            "total":{
+               "type":"long"
+            }
+         }
+      }
+   }
+}'
+----------------------------------
+
+NOTE: If you run this command, you must replace `x-pack-test-password` with your
+actual password.
+
+////
+This mapping specifies the following qualities for the data set:
+
+* The _@timestamp_ field is a date.
+//that uses the ISO format `epoch_second`,
+//which is the number of seconds since the epoch.
+* The _accept_, _deny_, and _total_ fields are long numbers.
+* The _host
+////
+
+You can then use the {es} `bulk` API to load the data set. The
+`upload_server-metrics.sh` script runs commands similar to the following
+example, which loads the four JSON files:
+
+[source,shell]
+----------------------------------
+
+curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
+http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_1.json"
+
+curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
+http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_2.json"
+
+curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
+http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_3.json"
+
+curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
+http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_4.json"
+----------------------------------
+
+TIP: This will upload 200MB of data. This is split into 4 files as there is a
+maximum 100MB limit when using the `_bulk` API.
+
+These commands might take some time to run, depending on the computing resources
+available.
+
+You can verify that the data was loaded successfully with the following command:
+
+[source,shell]
+----------------------------------
+
+curl 'http://localhost:9200/_cat/indices?v' -u elastic:x-pack-test-password
+----------------------------------
+
+You should see output similar to the following:
+
+[source,shell]
+----------------------------------
+
+health status index ... pri rep docs.count  ...
+green  open   server-metrics ... 1 0 905940  ...
+----------------------------------
+
+Next, you must define an index pattern for this data set:
+
+. Open {kib} in your web browser and log in. If you are running {kib}
+locally, go to `http://localhost:5601/`.
+
+. Click the **Management** tab, then **{kib}** > **Index Patterns**.
+
+. If you already have index patterns, click **Create Index** to define a new
+one. Otherwise, the **Create index pattern** wizard is already open.
+
+. For this tutorial, any pattern that matches the name of the index you've
+loaded will work. For example, enter `server-metrics*` as the index pattern.
+
+. In the **Configure settings** step, select the `@timestamp` field in the
+**Time Filter field name** list.
+
+. Click **Create index pattern**.
+
+This data set can now be analyzed in {ml} jobs in {kib}.

docs/en/ml/getting-started-multi.asciidoc

@@ -32,26 +32,12 @@ To create a multi-metric job in {kib}:
 . Open {kib} in your web browser and log in. If you are running {kib} locally,
 go to `http://localhost:5601/`.
 
-. Click **Machine Learning** in the side navigation, then click **Create new job**. +
-+
---
-[role="screenshot"]
-image::images/ml-kibana.jpg[Job Management]
---
+. Click **Machine Learning** in the side navigation, then click **Create new job**.
 
-. Click **Create multi metric job**. +
-+
---
-[role="screenshot"]
-image::images/ml-create-job2.jpg["Create a multi metric job"]
---
+. Select the index pattern that you created for the sample data.  For example,
+`server-metrics*`.
 
-. Click the `server-metrics` index. +
-+
---
-[role="screenshot"]
-image::images/ml-gs-index.jpg["Select an index"]
---
+. In the **Use a wizard** section, click **Multi metric**.
 
 . Configure the job by providing the following job settings: +
 +

docs/en/ml/getting-started-single.asciidoc

@@ -0,0 +1,331 @@
+[[ml-gs-jobs]]
+=== Creating Single Metric Jobs
+
+At this point in the tutorial, the goal is to detect anomalies in the
+total requests received by your applications and services. The sample data
+contains a single key performance indicator(KPI) to track this, which is the total
+requests over time. It is therefore logical to start by creating a single metric
+job for this KPI.
+
+TIP: If you are using aggregated data, you can create an advanced job
+and configure it to use a `summary_count_field_name`. The {ml} algorithms will
+make the best possible use of summarized data in this case. For simplicity, in
+this tutorial we will not make use of that advanced functionality. For more
+information, see <<ml-configuring-aggregation>>.
+
+A single metric job contains a single _detector_. A detector defines the type of
+analysis that will occur (for example, `max`, `average`, or `rare` analytical
+functions) and the fields that will be analyzed.
+
+To create a single metric job in {kib}:
+
+. Open {kib} in your web browser and log in. If you are running {kib} locally,
+go to `http://localhost:5601/`.
+
+. Click **Machine Learning** in the side navigation.
+
+. Click **Create new job**.
+
+. Select the index pattern that you created for the sample data.  For example,
+`server-metrics*`.
+
+. In the **Use a wizard** section, click **Single metric**.
+
+. Configure the job by providing the following information: +
++
+--
+[role="screenshot"]
+image::images/ml-gs-single-job.jpg["Create a new job from the server-metrics index"]
+--
+
+.. For the **Aggregation**, select `Sum`. This value specifies the analysis
+function that is used.
++
+--
+Some of the analytical functions look for single anomalous data points. For
+example, `max` identifies the maximum value that is seen within a bucket.
+Others perform some aggregation over the length of the bucket. For example,
+`mean` calculates the mean of all the data points seen within the bucket.
+Similarly, `count` calculates the total number of data points within the bucket.
+In this tutorial, you are using the `sum` function, which calculates the sum of
+the specified field's values within the bucket. For descriptions of all the
+functions, see <<ml-functions>>.
+--
+
+.. For the **Field**, select `total`. This value specifies the field that
+the detector uses in the function.
++
+--
+NOTE: Some functions such as `count` and `rare` do not require fields.
+--
+
+.. For the **Bucket span**, enter `10m`. This value specifies the size of the
+interval that the analysis is aggregated into.
++
+--
+The {xpackml} features use the concept of a bucket to divide up the time series
+into batches for processing. For example, if you are monitoring
+the total number of requests in the system,
+using a bucket span of 1 hour would mean that at the end of each hour, it
+calculates the sum of the requests for the last hour and computes the
+anomalousness of that value compared to previous hours.
+
+The bucket span has two purposes: it dictates over what time span to look for
+anomalous features in data, and also determines how quickly anomalies can be
+detected. Choosing a shorter bucket span enables anomalies to be detected more
+quickly. However, there is a risk of being too sensitive to natural variations
+or noise in the input data. Choosing too long a bucket span can mean that
+interesting anomalies are averaged away. There is also the possibility that the
+aggregation might smooth out some anomalies based on when the bucket starts
+in time.
+
+The bucket span has a significant impact on the analysis. When you're trying to
+determine what value to use, take into account the granularity at which you
+want to perform the analysis, the frequency of the input data, the duration of
+typical anomalies, and the frequency at which alerting is required.
+--
+
+. Determine whether you want to process all of the data or only part of it. If
+you want to analyze all of the existing data, click
+**Use full server-metrics* data**. If you want to see what happens when you
+stop and start {dfeeds} and process additional data over time, click the time
+picker in the {kib} toolbar. Since the sample data spans a period of time
+between March 23, 2017 and April 22, 2017, click **Absolute**. Set the start
+time to March 23, 2017 and the end time to April 1, 2017, for example. Once
+you've got the time range set up, click the **Go** button. +
++
+--
+[role="screenshot"]
+image::images/ml-gs-job1-time.jpg["Setting the time range for the {dfeed}"]
+--
++
+--
+A graph is generated, which represents the total number of requests over time.
+
+Note that the **Estimate bucket span** option is no longer greyed out in the
+**Buck span** field. This is an experimental feature that you can use to help
+determine an appropriate bucket span for your data. For the purposes of this
+tutorial, we will leave the bucket span at 10 minutes.
+--
+
+. Provide a name for the job, for example `total-requests`. The job name must
+be unique in your cluster. You can also optionally provide a description of the
+job and create a job group.
+
+. Click **Create Job**. +
++
+--
+[role="screenshot"]
+image::images/ml-gs-job1.jpg["A graph of the total number of requests over time"]
+--
+
+As the job is created, the graph is updated to give a visual representation of
+the progress of {ml} as the data is processed. This view is only available whilst the
+job is running.
+
+When the job is created, you can choose to view the results, continue the job
+in real-time, and create a watch. In this tutorial, we will look at how to
+manage jobs and {dfeeds} before we view the results.
+
+TIP: The `create_single_metic.sh` script creates a similar job and {dfeed} by
+using the {ml} APIs. You can download that script by clicking
+here: https://download.elastic.co/demos/machine_learning/gettingstarted/create_single_metric.sh[create_single_metric.sh]
+For API reference information, see {ref}/ml-apis.html[Machine Learning APIs].
+
+[[ml-gs-job1-manage]]
+=== Managing Jobs
+
+After you create a job, you can see its status in the **Job Management** tab: +
+
+[role="screenshot"]
+image::images/ml-gs-job1-manage1.jpg["Status information for the total-requests job"]
+
+The following information is provided for each job:
+
+Job ID::
+The unique identifier for the job.
+
+Description::
+The optional description of the job.
+
+Processed records::
+The number of records that have been processed by the job.
+
+Memory status::
+The status of the mathematical models. When you create jobs by using the APIs or
+by using the advanced options in {kib}, you can specify a `model_memory_limit`.
+That value is the maximum amount of memory resources that the mathematical
+models can use. Once that limit is approached, data pruning becomes more
+aggressive. Upon exceeding that limit, new entities are not modeled. For more
+information about this setting, see
+{ref}/ml-job-resource.html#ml-apilimits[Analysis Limits]. The memory status
+field reflects whether you have reached or exceeded the model memory limit. It
+can have one of the following values: +
+`ok`::: The models stayed below the configured value.
+`soft_limit`::: The models used more than 60% of the configured memory limit
+and older unused models will be pruned to free up space.
+`hard_limit`::: The models used more space than the configured memory limit.
+As a result, not all incoming data was processed.
+
+Job state::
+The status of the job, which can be one of the following values: +
+`opened`::: The job is available to receive and process data.
+`closed`::: The job finished successfully with its model state persisted.
+The job must be opened before it can accept further data.
+`closing`::: The job close action is in progress and has not yet completed.
+A closing job cannot accept further data.
+`failed`::: The job did not finish successfully due to an error.
+This situation can occur due to invalid input data.
+If the job had irrevocably failed, it must be force closed and then deleted.
+If the {dfeed} can be corrected, the job can be closed and then re-opened.
+
+{dfeed-cap} state::
+The status of the {dfeed}, which can be one of the following values: +
+started::: The {dfeed} is actively receiving data.
+stopped::: The {dfeed} is stopped and will not receive data until it is
+re-started.
+
+Latest timestamp::
+The timestamp of the last processed record.
+
+
+If you click the arrow beside the name of job, you can show or hide additional
+information, such as the settings, configuration information, or messages for
+the job.
+
+You can also click one of the **Actions** buttons to start the {dfeed}, edit
+the job or {dfeed}, and clone or delete the job, for example.
+
+[float]
+[[ml-gs-job1-datafeed]]
+==== Managing {dfeeds-cap}
+
+A {dfeed} can be started and stopped multiple times throughout its lifecycle.
+If you want to retrieve more data from {es} and the {dfeed} is stopped, you must
+restart it.
+
+For example, if you did not use the full data when you created the job, you can
+now process the remaining data by restarting the {dfeed}:
+
+. In the **Machine Learning** / **Job Management** tab, click the following
+button to start the {dfeed}: image:images/ml-start-feed.jpg["Start {dfeed}"]
+
+
+. Choose a start time and end time. For example,
+click **Continue from 2017-04-01 23:59:00** and select **2017-04-30** as the
+search end time. Then click **Start**. The date picker defaults to the latest
+timestamp of processed data. Be careful not to leave any gaps in the analysis,
+otherwise you might miss anomalies. +
++
+--
+[role="screenshot"]
+image::images/ml-gs-job1-datafeed.jpg["Restarting a {dfeed}"]
+--
+
+The {dfeed} state changes to `started`, the job state changes to `opened`,
+and the number of processed records increases as the new data is analyzed. The
+latest timestamp information also increases.
+
+TIP: If your data is being loaded continuously, you can continue running the job
+in real time. For this, start your {dfeed} and select **No end time**.
+
+If you want to stop the {dfeed} at this point, you can click the following
+button: image:images/ml-stop-feed.jpg["Stop {dfeed}"]
+
+Now that you have processed all the data, let's start exploring the job results.
+
+[[ml-gs-job1-analyze]]
+=== Exploring Single Metric Job Results
+
+The {xpackml} features analyze the input stream of data, model its behavior,
+and perform analysis based on the detectors you defined in your job. When an
+event occurs outside of the model, that event is identified as an anomaly.
+
+Result records for each anomaly are stored in `.ml-anomalies-*` indices in {es}.
+By default, the name of the index where {ml} results are stored is labelled
+`shared`, which corresponds to the `.ml-anomalies-shared` index.
+
+You can use the **Anomaly Explorer** or the **Single Metric Viewer** in {kib} to
+view the analysis results.
+
+Anomaly Explorer::
+  This view contains swim lanes showing the maximum anomaly score over time.
+  There is an overall swim lane that shows the overall score for the job, and
+  also swim lanes for each influencer. By selecting a block in a swim lane, the
+  anomaly details are displayed alongside the original source data (where
+  applicable).
+
+Single Metric Viewer::
+  This view contains a chart that represents the actual and expected values over
+  time. This is only available for jobs that analyze a single time series and
+  where `model_plot_config` is enabled. As in the **Anomaly Explorer**, anomalous
+  data points are shown in different colors depending on their score.
+
+By default when you view the results for a single metric job, the
+**Single Metric Viewer** opens:
+[role="screenshot"]
+image::images/ml-gs-job1-analysis.jpg["Single Metric Viewer for total-requests job"]
+
+
+The blue line in the chart represents the actual data values. The shaded blue
+area represents the bounds for the expected values. The area between the upper
+and lower bounds are the most likely values for the model. If a value is outside
+of this area then it can be said to be anomalous.
+
+If you slide the time selector from the beginning of the data to the end of the
+data, you can see how the model improves as it processes more data. At the
+beginning, the expected range of values is pretty broad and the model is not
+capturing the periodicity in the data. But it quickly learns and begins to
+reflect the daily variation.
+
+Any data points outside the range that was predicted by the model are marked
+as anomalies. When you have high volumes of real-life data, many anomalies
+might be found. These vary in probability from very likely to highly unlikely,
+that is to say, from not particularly anomalous to highly anomalous. There
+can be none, one or two or tens, sometimes hundreds of anomalies found within
+each bucket. There can be many thousands found per job. In order to provide
+a sensible view of the results, an _anomaly score_ is calculated for each bucket
+time interval. The anomaly score is a value from 0 to 100, which indicates
+the significance of the observed anomaly compared to previously seen anomalies.
+The highly anomalous values are shown in red and the low scored values are
+indicated in blue. An interval with a high anomaly score is significant and
+requires investigation.
+
+Slide the time selector to a section of the time series that contains a red
+anomaly data point. If you hover over the point, you can see more information
+about that data point. You can also see details in the **Anomalies** section
+of the viewer. For example:
+[role="screenshot"]
+image::images/ml-gs-job1-anomalies.jpg["Single Metric Viewer Anomalies for total-requests job"]
+
+For each anomaly you can see key details such as the time, the actual and
+expected ("typical") values, and their probability.
+
+By default, the table contains all anomalies that have a severity of "warning"
+or higher in the selected section of the timeline. If you are only interested in
+critical anomalies, for example, you can change the severity threshold for this
+table.
+
+The anomalies table also automatically calculates an interval for the data in
+the table. If the time difference between the earliest and latest records in the
+table is less than two days, the data is aggregated by hour to show the details
+of the highest severity anomaly for each detector.  Otherwise, it is
+aggregated by day. You can change the interval for the table, for example, to
+show all anomalies.
+
+You can see the same information in a different format by using the
+**Anomaly Explorer**:
+[role="screenshot"]
+image::images/ml-gs-job1-explorer.jpg["Anomaly Explorer for total-requests job"]
+
+
+Click one of the red sections in the swim lane to see details about the anomalies
+that occurred in that time interval. For example:
+[role="screenshot"]
+image::images/ml-gs-job1-explorer-anomaly.jpg["Anomaly Explorer details for total-requests job"]
+
+After you have identified anomalies, often the next step is to try to determine
+the context of those situations. For example, are there other factors that are
+contributing to the problem? Are the anomalies confined to particular
+applications or servers? You can begin to troubleshoot these situations by
+layering additional jobs or creating multi-metric jobs.

docs/en/ml/getting-started-wizards.asciidoc

@@ -0,0 +1,99 @@
+[[ml-gs-wizards]]
+=== Creating Jobs in {kib}
+++++
+<titleabbrev>Creating Jobs</titleabbrev>
+++++
+
+Machine learning jobs contain the configuration information and metadata
+necessary to perform an analytical task. They also contain the results of the
+analytical task.
+
+[NOTE]
+--
+This tutorial uses {kib} to create jobs and view results, but you can
+alternatively use APIs to accomplish most tasks.
+For API reference information, see {ref}/ml-apis.html[Machine Learning APIs].
+
+The {xpackml} features in {kib} use pop-ups. You must configure your
+web browser so that it does not block pop-up windows or create an
+exception for your {kib} URL.
+--
+
+{kib} provides wizards that help you create typical {ml} jobs. For example, you
+can use wizards to create single metric, multi-metric, population, and advanced
+jobs.
+
+To see the job creation wizards:
+
+. Open {kib} in your web browser and log in. If you are running {kib} locally,
+go to `http://localhost:5601/`.
+
+. Click **Machine Learning** in the side navigation.
+
+. Click **Create new job**.
+
+. Click the `server-metrics*` index pattern.
+
+You can then choose from a list of job wizards. For example:
+
+[role="screenshot"]
+image::images/ml-create-job.jpg["Job creation wizards in {kib}"]
+
+If you are not certain which wizard to use, there is also a **Data Visualizer**
+that can help you explore the fields in your data.
+
+To learn more about the sample data:
+
+. Click **Data Visualizer**. +
++
+--
+[role="screenshot"]
+image::images/ml-data-visualizer.jpg["Data Visualizer in {kib}"]
+--
+
+. Select a time period that you're interested in exploring by using the time
+picker in the {kib} toolbar. Alternatively, click
+**Use full server-metrics* data** to view data over the full time range. In this
+sample data, the documents relate to March and April 2017.
+
+. Optional: Change the number of documents per shard that are used in the
+visualizations. There is a relatively small number of documents in the sample
+data, so you can choose a value of `all`. For larger data sets, keep in mind
+that using a large sample size increases query run times and increases the load
+on the cluster.
+
+[role="screenshot"]
+image::images/ml-data-metrics.jpg["Data Visualizer output for metrics in {kib}"]
+
+The fields in the indices are listed in two sections.  The first section contains
+the numeric ("metric") fields. The second section contains non-metric fields
+(such as `keyword`, `text`, `date`, `boolean`, `ip`, and `geo_point` data types).
+
+For metric fields, the **Data Visualizer** indicates how many documents contain
+the field in the selected time period. It also provides information about the
+minimum, median, and maximum values, the number of distinct values, and their
+distribution. You can use the distribution chart to get a better idea of how
+the values in the data are clustered. Alternatively, you can view the top values
+for metric fields. For example:
+
+[role="screenshot"]
+image::images/ml-data-topmetrics.jpg["Data Visualizer output for top values in {kib}"]
+
+For date fields, the **Data Visualizer** provides the earliest and latest field
+values and the number and percentage of documents that contain the field
+during the selected time period. For example:
+
+[role="screenshot"]
+image::images/ml-data-dates.jpg["Data Visualizer output for date fields in {kib}"]
+
+For keyword fields, the **Data Visualizer** provides the number of distinct
+values, a list of the top values, and the number and percentage of documents
+that contain the field during the selected time period. For example:
+
+[role="screenshot"]
+image::images/ml-data-keywords.jpg["Data Visualizer output for date fields in {kib}"]
+
+In this tutorial, you will create single and multi-metric jobs that use the
+`total`, `response`, `service`, and `host` fields. Though there is an option to
+create an advanced job directly from the **Data Visualizer**, we will use the
+single and multi-metric job creation wizards instead.

docs/en/ml/getting-started.asciidoc

@@ -1,14 +1,9 @@
 [[ml-getting-started]]
-== Getting Started
+== Getting Started with Machine Learning
+++++
+<titleabbrev>Getting Started</titleabbrev>
+++++
 
-////
-{xpackml} features automatically detect:
-* Anomalies in single or multiple time series
-* Outliers in a population (also known as _entity profiling_)
-* Rare events (also known as _log categorization_)
-
-This tutorial is focuses on an anomaly detection scenario in single time series.
-////
 Ready to get some hands-on experience with the {xpackml} features? This
 tutorial shows you how to:
 
@@ -79,583 +74,8 @@ significant changes to the system. You can alternatively assign the
 
 For more information, see <<built-in-roles>> and <<privileges-list-cluster>>.
 
-[[ml-gs-data]]
-=== Identifying Data for Analysis
-
-For the purposes of this tutorial, we provide sample data that you can play with
-and search in {es}. When you consider your own data, however, it's important to
-take a moment and think about where the {xpackml} features will be most
-impactful.
-
-The first consideration is that it must be time series data. The {ml} features
-are designed to model and detect anomalies in time series data.
-
-The second consideration, especially when you are first learning to use {ml},
-is the importance of the data and how familiar you are with it. Ideally, it is
-information that contains key performance indicators (KPIs) for the health,
-security, or success of your business or system. It is information that you need
-to monitor and act on when anomalous behavior occurs. You might even have {kib}
-dashboards that you're already using to watch this data. The better you know the
-data, the quicker you will be able to create {ml} jobs that generate useful
-insights.
-
-The final consideration is where the data is located. This tutorial assumes that
-your data is stored in {es}. It guides you through the steps required to create
-a _{dfeed}_ that passes data to a job. If your own data is outside of {es},
-analysis is still possible by using a post data API.
-
-IMPORTANT: If you want to create {ml} jobs in {kib}, you must use {dfeeds}.
-That is to say, you must store your input data in {es}. When you create
-a job, you select an existing index pattern and {kib} configures the {dfeed}
-for you under the covers.
-
-
-[float]
-[[ml-gs-sampledata]]
-==== Obtaining a Sample Data Set
-
-In this step we will upload some sample data to {es}. This is standard
-{es} functionality, and is needed to set the stage for using {ml}.
-
-The sample data for this tutorial contains information about the requests that
-are received by various applications and services in a system. A system
-administrator might use this type of information to track the total number of
-requests across all of the infrastructure. If the number of requests increases
-or decreases unexpectedly, for example, this might be an indication that there
-is a problem or that resources need to be redistributed. By using the {xpack}
-{ml} features to model the behavior of this data, it is easier to identify
-anomalies and take appropriate action.
-
-Download this sample data by clicking here:
-https://download.elastic.co/demos/machine_learning/gettingstarted/server_metrics.tar.gz[server_metrics.tar.gz]
-
-Use the following commands to extract the files:
-
-[source,shell]
-----------------------------------
-tar -zxvf server_metrics.tar.gz
-----------------------------------
-
-Each document in the server-metrics data set has the following schema:
-
-[source,js]
-----------------------------------
-{
-  "index":
-  {
-    "_index":"server-metrics",
-    "_type":"metric",
-    "_id":"1177"
-  }
-}
-{
-  "@timestamp":"2017-03-23T13:00:00",
-  "accept":36320,
-  "deny":4156,
-  "host":"server_2",
-  "response":2.4558210155,
-  "service":"app_3",
-  "total":40476
-}
-----------------------------------
-
-TIP: The sample data sets include summarized data. For example, the `total`
-value is a sum of the requests that were received by a specific service at a
-particular time. If your data is stored in {es}, you can generate
-this type of sum or average by using aggregations. One of the benefits of
-summarizing data this way is that {es} automatically distributes
-these calculations across your cluster. You can then feed this summarized data
-into {xpackml} instead of raw results, which reduces the volume
-of data that must be considered while detecting anomalies. For the purposes of
-this tutorial, however, these summary values are stored in {es}. For more
-information, see <<ml-configuring-aggregation>>.
-
-Before you load the data set, you need to set up {ref}/mapping.html[_mappings_]
-for the fields. Mappings divide the documents in the index into logical groups
-and specify a field's characteristics, such as the field's searchability or
-whether or not it's _tokenized_, or broken up into separate words.
-
-The sample data includes an `upload_server-metrics.sh` script, which you can use
-to create the mappings and load the data set. You can download it by clicking
-here: https://download.elastic.co/demos/machine_learning/gettingstarted/upload_server-metrics.sh[upload_server-metrics.sh]
-Before you run it, however, you must edit the USERNAME and PASSWORD variables
-with your actual user ID and password.
-
-The script runs a command similar to the following example, which sets up a
-mapping for the data set:
-
-[source,shell]
-----------------------------------
-
-curl -u elastic:x-pack-test-password -X PUT -H 'Content-Type: application/json'
-http://localhost:9200/server-metrics -d '{
-   "settings":{
-      "number_of_shards":1,
-      "number_of_replicas":0
-   },
-   "mappings":{
-      "metric":{
-         "properties":{
-            "@timestamp":{
-               "type":"date"
-            },
-            "accept":{
-               "type":"long"
-            },
-            "deny":{
-               "type":"long"
-            },
-            "host":{
-               "type":"keyword"
-            },
-            "response":{
-               "type":"float"
-            },
-            "service":{
-               "type":"keyword"
-            },
-            "total":{
-               "type":"long"
-            }
-         }
-      }
-   }
-}'
-----------------------------------
-
-NOTE: If you run this command, you must replace `x-pack-test-password` with your
-actual password.
-
-////
-This mapping specifies the following qualities for the data set:
-
-* The _@timestamp_ field is a date.
-//that uses the ISO format `epoch_second`,
-//which is the number of seconds since the epoch.
-* The _accept_, _deny_, and _total_ fields are long numbers.
-* The _host
-////
-
-You can then use the {es} `bulk` API to load the data set. The
-`upload_server-metrics.sh` script runs commands similar to the following
-example, which loads the four JSON files:
-
-[source,shell]
-----------------------------------
-
-curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
-http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_1.json"
-
-curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
-http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_2.json"
-
-curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
-http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_3.json"
-
-curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
-http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_4.json"
-----------------------------------
-
-TIP: This will upload 200MB of data. This is split into 4 files as there is a
-maximum 100MB limit when using the `_bulk` API.
-
-These commands might take some time to run, depending on the computing resources
-available.
-
-You can verify that the data was loaded successfully with the following command:
-
-[source,shell]
-----------------------------------
-
-curl 'http://localhost:9200/_cat/indices?v' -u elastic:x-pack-test-password
-----------------------------------
-
-You should see output similar to the following:
-
-[source,shell]
-----------------------------------
-
-health status index ... pri rep docs.count  ...
-green  open   server-metrics ... 1 0 905940  ...
-----------------------------------
-
-Next, you must define an index pattern for this data set:
-
-. Open {kib} in your web browser and log in. If you are running {kib}
-locally, go to `http://localhost:5601/`.
-
-. Click the **Management** tab, then **Index Patterns**.
-
-. If you already have index patterns, click the plus sign (+) to define a new
-one. Otherwise, the **Configure an index pattern** wizard is already open.
-
-. For this tutorial, any pattern that matches the name of the index you've
-loaded will work. For example, enter `server-metrics*` as the index pattern.
-
-. Verify that the **Index contains time-based events** is checked.
-
-. Select the `@timestamp` field from the **Time-field name** list.
-
-. Click **Create**.
-
-This data set can now be analyzed in {ml} jobs in {kib}.
-
-
-[[ml-gs-jobs]]
-=== Creating Single Metric Jobs
-
-Machine learning jobs contain the configuration information and metadata
-necessary to perform an analytical task. They also contain the results of the
-analytical task.
-
-[NOTE]
---
-This tutorial uses {kib} to create jobs and view results, but you can
-alternatively use APIs to accomplish most tasks.
-For API reference information, see {ref}/ml-apis.html[Machine Learning APIs].
-
-The {xpackml} features in {kib} use pop-ups. You must configure your
-web browser so that it does not block pop-up windows or create an
-exception for your Kibana URL.
---
-
-You can choose to create single metric, multi-metric, or advanced jobs in
-{kib}. At this point in the tutorial, the goal is to detect anomalies in the
-total requests received by your applications and services. The sample data
-contains a single key performance indicator to track this, which is the total
-requests over time. It is therefore logical to start by creating a single metric
-job for this KPI.
-
-TIP: If you are using aggregated data, you can create an advanced job
-and configure it to use a `summary_count_field_name`. The {ml} algorithms will
-make the best possible use of summarized data in this case. For simplicity, in
-this tutorial we will not make use of that advanced functionality.
-
-//TO-DO: Add link to aggregations.asciidoc: For more information, see <<>>.
-
-A single metric job contains a single _detector_. A detector defines the type of
-analysis that will occur (for example, `max`, `average`, or `rare` analytical
-functions) and the fields that will be analyzed.
-
-To create a single metric job in {kib}:
-
-. Open {kib} in your web browser and log in. If you are running {kib} locally,
-go to `http://localhost:5601/`.
-
-. Click **Machine Learning** in the side navigation: +
-+
---
-[role="screenshot"]
-image::images/ml-kibana.jpg[Job Management]
---
-
-. Click **Create new job**.
-
-. Click **Create single metric job**. +
-+
---
-[role="screenshot"]
-image::images/ml-create-jobs.jpg["Create a new job"]
---
-
-. Click the `server-metrics` index. +
-+
---
-[role="screenshot"]
-image::images/ml-gs-index.jpg["Select an index"]
---
-
-. Configure the job by providing the following information: +
-+
---
-[role="screenshot"]
-image::images/ml-gs-single-job.jpg["Create a new job from the server-metrics index"]
---
-
-.. For the **Aggregation**, select `Sum`. This value specifies the analysis
-function that is used.
-+
---
-Some of the analytical functions look for single anomalous data points. For
-example, `max` identifies the maximum value that is seen within a bucket.
-Others perform some aggregation over the length of the bucket. For example,
-`mean` calculates the mean of all the data points seen within the bucket.
-Similarly, `count` calculates the total number of data points within the bucket.
-In this tutorial, you are using the `sum` function, which calculates the sum of
-the specified field's values within the bucket.
---
-
-.. For the **Field**, select `total`. This value specifies the field that
-the detector uses in the function.
-+
---
-NOTE: Some functions such as `count` and `rare` do not require fields.
---
-
-.. For the **Bucket span**, enter `10m`. This value specifies the size of the
-interval that the analysis is aggregated into.
-+
---
-The {xpackml} features use the concept of a bucket to divide up the time series
-into batches for processing. For example, if you are monitoring
-the total number of requests in the system,
-//and receive a data point every 10 minutes
-using a bucket span of 1 hour would mean that at the end of each hour, it
-calculates the sum of the requests for the last hour and computes the
-anomalousness of that value compared to previous hours.
-
-The bucket span has two purposes: it dictates over what time span to look for
-anomalous features in data, and also determines how quickly anomalies can be
-detected. Choosing a shorter bucket span enables anomalies to be detected more
-quickly. However, there is a risk of being too sensitive to natural variations
-or noise in the input data. Choosing too long a bucket span can mean that
-interesting anomalies are averaged away. There is also the possibility that the
-aggregation might smooth out some anomalies based on when the bucket starts
-in time.
-
-The bucket span has a significant impact on the analysis. When you're trying to
-determine what value to use, take into account the granularity at which you
-want to perform the analysis, the frequency of the input data, the duration of
-typical anomalies and the frequency at which alerting is required.
---
-
-. Determine whether you want to process all of the data or only part of it. If
-you want to analyze all of the existing data, click
-**Use full server-metrics* data**. If you want to see what happens when you
-stop and start {dfeeds} and process additional data over time, click the time
-picker in the {kib} toolbar. Since the sample data spans a period of time
-between March 23, 2017 and April 22, 2017, click **Absolute**. Set the start
-time to March 23, 2017 and the end time to April 1, 2017, for example. Once
-you've got the time range set up, click the **Go** button. +
-+
---
-[role="screenshot"]
-image::images/ml-gs-job1-time.jpg["Setting the time range for the {dfeed}"]
---
-+
---
-A graph is generated, which represents the total number of requests over time.
---
-
-. Provide a name for the job, for example `total-requests`. The job name must
-be unique in your cluster. You can also optionally provide a description of the
-job.
-
-. Click **Create Job**. +
-+
---
-[role="screenshot"]
-image::images/ml-gs-job1.jpg["A graph of the total number of requests over time"]
---
-
-As the job is created, the graph is updated to give a visual representation of
-the progress of {ml} as the data is processed. This view is only available whilst the
-job is running.
-
-TIP: The `create_single_metic.sh` script creates a similar job and {dfeed} by
-using the {ml} APIs. You can download that script by clicking
-here: https://download.elastic.co/demos/machine_learning/gettingstarted/create_single_metric.sh[create_single_metric.sh]
-For API reference information, see {ref}/ml-apis.html[Machine Learning APIs].
-
-[[ml-gs-job1-manage]]
-=== Managing Jobs
-
-After you create a job, you can see its status in the **Job Management** tab: +
-
-[role="screenshot"]
-image::images/ml-gs-job1-manage1.jpg["Status information for the total-requests job"]
-
-The following information is provided for each job:
-
-Job ID::
-The unique identifier for the job.
-
-Description::
-The optional description of the job.
-
-Processed records::
-The number of records that have been processed by the job.
-
-Memory status::
-The status of the mathematical models. When you create jobs by using the APIs or
-by using the advanced options in {kib}, you can specify a `model_memory_limit`.
-That value is the maximum amount of memory resources that the mathematical
-models can use. Once that limit is approached, data pruning becomes more
-aggressive. Upon exceeding that limit, new entities are not modeled. For more
-information about this setting, see
-{ref}/ml-job-resource.html#ml-apilimits[Analysis Limits]. The memory status
-field reflects whether you have reached or exceeded the model memory limit. It
-can have one of the following values: +
-`ok`::: The models stayed below the configured value.
-`soft_limit`::: The models used more than 60% of the configured memory limit
-and older unused models will be pruned to free up space.
-`hard_limit`::: The models used more space than the configured memory limit.
-As a result, not all incoming data was processed.
-
-Job state::
-The status of the job, which can be one of the following values: +
-`open`::: The job is available to receive and process data.
-`closed`::: The job finished successfully with its model state persisted.
-The job must be opened before it can accept further data.
-`closing`::: The job close action is in progress and has not yet completed.
-A closing job cannot accept further data.
-`failed`::: The job did not finish successfully due to an error.
-This situation can occur due to invalid input data.
-If the job had irrevocably failed, it must be force closed and then deleted.
-If the {dfeed} can be corrected, the job can be closed and then re-opened.
-
-{dfeed-cap} state::
-The status of the {dfeed}, which can be one of the following values: +
-started::: The {dfeed} is actively receiving data.
-stopped::: The {dfeed} is stopped and will not receive data until it is
-re-started.
-
-Latest timestamp::
-The timestamp of the last processed record.
-
-
-If you click the arrow beside the name of job, you can show or hide additional
-information, such as the settings, configuration information, or messages for
-the job.
-
-You can also click one of the **Actions** buttons to start the {dfeed}, edit
-the job or {dfeed}, and clone or delete the job, for example.
-
-[float]
-[[ml-gs-job1-datafeed]]
-==== Managing {dfeeds-cap}
-
-A {dfeed} can be started and stopped multiple times throughout its lifecycle.
-If you want to retrieve more data from {es} and the {dfeed} is stopped, you must
-restart it.
-
-For example, if you did not use the full data when you created the job, you can
-now process the remaining data by restarting the {dfeed}:
-
-. In the **Machine Learning** / **Job Management** tab, click the following
-button to start the {dfeed}: image:images/ml-start-feed.jpg["Start {dfeed}"]
-
-
-. Choose a start time and end time. For example,
-click **Continue from 2017-04-01 23:59:00** and select **2017-04-30** as the
-search end time. Then click **Start**. The date picker defaults to the latest
-timestamp of processed data. Be careful not to leave any gaps in the analysis,
-otherwise you might miss anomalies. +
-+
---
-[role="screenshot"]
-image::images/ml-gs-job1-datafeed.jpg["Restarting a {dfeed}"]
---
-
-The {dfeed} state changes to `started`, the job state changes to `opened`,
-and the number of processed records increases as the new data is analyzed. The
-latest timestamp information also increases. For example:
-[role="screenshot"]
-image::images/ml-gs-job1-manage2.jpg["Job opened and {dfeed} started"]
-
-TIP: If your data is being loaded continuously, you can continue running the job
-in real time. For this, start your {dfeed} and select **No end time**.
-
-If you want to stop the {dfeed} at this point, you can click the following
-button: image:images/ml-stop-feed.jpg["Stop {dfeed}"]
-
-Now that you have processed all the data, let's start exploring the job results.
-
-[[ml-gs-job1-analyze]]
-=== Exploring Single Metric Job Results
-
-The {xpackml} features analyze the input stream of data, model its behavior,
-and perform analysis based on the detectors you defined in your job. When an
-event occurs outside of the model, that event is identified as an anomaly.
-
-Result records for each anomaly are stored in `.ml-anomalies-*` indices in {es}.
-By default, the name of the index where {ml} results are stored is labelled
-`shared`, which corresponds to the `.ml-anomalies-shared` index.
-
-You can use the **Anomaly Explorer** or the **Single Metric Viewer** in {kib} to
-view the analysis results.
-
-Anomaly Explorer::
-  This view contains swim lanes showing the maximum anomaly score over time.
-  There is an overall swim lane that shows the overall score for the job, and
-  also swim lanes for each influencer. By selecting a block in a swim lane, the
-  anomaly details are displayed alongside the original source data (where
-  applicable).
-
-Single Metric Viewer::
-  This view contains a chart that represents the actual and expected values over
-  time. This is only available for jobs that analyze a single time series and
-  where `model_plot_config` is enabled. As in the **Anomaly Explorer**, anomalous
-  data points are shown in different colors depending on their score.
-
-By default when you view the results for a single metric job, the
-**Single Metric Viewer** opens:
-[role="screenshot"]
-image::images/ml-gs-job1-analysis.jpg["Single Metric Viewer for total-requests job"]
-
-
-The blue line in the chart represents the actual data values. The shaded blue
-area represents the bounds for the expected values. The area between the upper
-and lower bounds are the most likely values for the model. If a value is outside
-of this area then it can be said to be anomalous.
-
-If you slide the time selector from the beginning of the data to the end of the
-data, you can see how the model improves as it processes more data. At the
-beginning, the expected range of values is pretty broad and the model is not
-capturing the periodicity in the data. But it quickly learns and begins to
-reflect the daily variation.
-
-Any data points outside the range that was predicted by the model are marked
-as anomalies. When you have high volumes of real-life data, many anomalies
-might be found. These vary in probability from very likely to highly unlikely,
-that is to say, from not particularly anomalous to highly anomalous. There
-can be none, one or two or tens, sometimes hundreds of anomalies found within
-each bucket. There can be many thousands found per job. In order to provide
-a sensible view of the results, an _anomaly score_ is calculated for each bucket
-time interval. The anomaly score is a value from 0 to 100, which indicates
-the significance of the observed anomaly compared to previously seen anomalies.
-The highly anomalous values are shown in red and the low scored values are
-indicated in blue. An interval with a high anomaly score is significant and
-requires investigation.
-
-Slide the time selector to a section of the time series that contains a red
-anomaly data point. If you hover over the point, you can see more information
-about that data point. You can also see details in the **Anomalies** section
-of the viewer. For example:
-[role="screenshot"]
-image::images/ml-gs-job1-anomalies.jpg["Single Metric Viewer Anomalies for total-requests job"]
-
-For each anomaly you can see key details such as the time, the actual and
-expected ("typical") values, and their probability.
-
-By default, the table contains all anomalies that have a severity of "warning"
-or higher in the selected section of the timeline. If you are only interested in
-critical anomalies, for example, you can change the severity threshold for this
-table.
-
-The anomalies table also automatically calculates an interval for the data in
-the table. If the time difference between the earliest and latest records in the
-table is less than two days, the data is aggregated by hour to show the details
-of the highest severity anomaly for each detector.  Otherwise, it is
-aggregated by day. You can change the interval for the table, for example, to
-show all anomalies.
-
-You can see the same information in a different format by using the
-**Anomaly Explorer**:
-[role="screenshot"]
-image::images/ml-gs-job1-explorer.jpg["Anomaly Explorer for total-requests job"]
-
-
-Click one of the red sections in the swim lane to see details about the anomalies
-that occurred in that time interval. For example:
-[role="screenshot"]
-image::images/ml-gs-job1-explorer-anomaly.jpg["Anomaly Explorer details for total-requests job"]
-
-After you have identified anomalies, often the next step is to try to determine
-the context of those situations. For example, are there other factors that are
-contributing to the problem? Are the anomalies confined to particular
-applications or servers? You can begin to troubleshoot these situations by
-layering additional jobs or creating multi-metric jobs.
-
+include::getting-started-data.asciidoc[]
+include::getting-started-wizards.asciidoc[]
+include::getting-started-single.asciidoc[]
 include::getting-started-multi.asciidoc[]
 include::getting-started-next.asciidoc[]

docs/en/release-notes/7.0.0-alpha1.asciidoc

@@ -9,6 +9,11 @@ Machine Learning::
 * The `max_running_jobs` node property is removed in this release. Use the
 `xpack.ml.max_open_jobs` setting instead. For more information, see <<ml-settings>>.
 
+Security::
+* The fields returned as part of the mappings section by get index, get
+mappings, get field mappings and field capabilities API are now only the
+ones that the user is authorized to access in case field level security is enabled.
+
 See also:
 
 * <<release-notes-7.0.0-alpha1,{es} 7.0.0-alpha1 Release Notes>>

docs/en/release-notes/xpack-breaking.asciidoc

@@ -26,6 +26,11 @@ Machine Learning::
 * The `max_running_jobs` node property is removed in this release. Use the
 `xpack.ml.max_open_jobs` setting instead. For more information, <<ml-settings>>.
 
+Security::
+* The fields returned as part of the mappings section by get index, get
+mappings, get field mappings and field capabilities API are now only the ones
+that the user is authorized to access in case field level security is enabled.
+
 See also:
 
 * <<breaking-changes-7.0,{es} Breaking Changes>>

docs/en/rest-api/watcher/execute-watch.asciidoc

@@ -118,6 +118,18 @@ are five possible modes an action can be associated with:
 You must have `manage_watcher` cluster privileges to use this API. For more
 information, see {xpack-ref}/security-privileges.html[Security Privileges].
 
+[float]
+==== Security Integration
+
+When {security} is enabled on your Elasticsearch cluster, then watches will be
+executed with the privileges of the user that stored the watches. If your user
+is allowed to read index `a`, but not index `b`, then the exact same set of
+rules will apply during execution of a watch.
+
+When using the execute watch API, the authorization data of the user that
+called the API will be used as a base, instead of of the information who stored
+the watch.
+
 [float]
 ==== Examples
 

docs/en/rest-api/watcher/put-watch.asciidoc

@@ -74,6 +74,14 @@ A watch has the following fields:
 You must have `manage_watcher` cluster privileges to use this API. For more
 information, see {xpack-ref}/security-privileges.html[Security Privileges].
 
+[float]
+==== Security Integration
+
+When {security} is enabled, your watch will only be able to index or search on
+indices for which the user that stored the watch, has privileges. If the user is
+able to read index `a`, but not index `b`, the same will apply, when the watch
+is executed.
+
 [float]
 ==== Examples
 

docs/en/security/tribe-clients-integrations/java.asciidoc

@@ -1,6 +1,8 @@
 [[java-clients]]
 === Java Client and Security
 
+deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.]
+
 {security} supports the Java http://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html[transport client] for Elasticsearch.
 The transport client uses the same transport protocol that the cluster nodes use
 for inter-node communication. It is very efficient as it does not have to marshall

docs/en/settings/ml-settings.asciidoc

@@ -38,7 +38,10 @@ IMPORTANT: If you want to use {ml} features in your cluster, you must have
 default behavior.
 
 `xpack.ml.max_open_jobs`::
-The maximum number of jobs that can run on a node. Defaults to `10`.
+The maximum number of jobs that can run on a node. Defaults to `20`.
+The maximum number of jobs is also constrained by memory usage, so fewer
+jobs than specified by this setting will run on a node if the estimated
+memory use of the jobs would be higher than allowed.
 
 `xpack.ml.max_machine_memory_percent`::
 The maximum percentage of the machine's memory that {ml} may use for running

docs/en/settings/monitoring-settings.asciidoc

@@ -72,15 +72,22 @@ Settings API.
 
 `xpack.monitoring.history.duration`::
 
-Sets the retention duration beyond which the indices created by a Monitoring exporter will
-be automatically deleted. Defaults to `7d` (7 days).
-+
-This setting has a minimum value of `1d` (1 day) to ensure that something is being monitored,
-and it cannot be disabled.
+Sets the retention duration beyond which the indices created by a Monitoring
+exporter are automatically deleted. Defaults to `7d` (7 days).
 +
+--
+This setting has a minimum value of `1d` (1 day) to ensure that something is
+being monitored, and it cannot be disabled.
+
 IMPORTANT: This setting currently only impacts `local`-type exporters. Indices created using
 the `http` exporter will not be deleted automatically.
 
+If both {monitoring} and {watcher} are enabled, you can use this setting to
+affect the {watcher} cleaner service too. For more information, see the
+`xpack.watcher.history.cleaner_service.enabled` setting in the 
+<<notification-settings>>.
+--
+
 `xpack.monitoring.exporters`::
 
 Configures where the agent stores monitoring data. By default, the agent uses a

docs/en/settings/notification-settings.asciidoc

@@ -35,9 +35,13 @@ required. For more information, see
 {xpack-ref}/encrypting-data.html[Encrypting sensitive data in {watcher}].
 
 `xpack.watcher.history.cleaner_service.enabled`::
-Set to `false` (default) to disable the cleaner service, which removes previous
-versions of {watcher} indices (for example, .watcher-history*) when it
-determines that they are old.
+Set to `false` (default) to disable the cleaner service. If this setting is
+`true`, the `xpack.monitoring.enabled` setting must also be set to `true`. The
+cleaner service removes previous versions of {watcher} indices (for example,
+`.watcher-history*`) when it determines that they are old. The duration of
+{watcher} indices is determined by the `xpack.monitoring.history.duration`
+setting, which defaults to 7 days. For more information about that setting,
+see <<monitoring-settings>>.
 
 `xpack.http.proxy.host`::
 Specifies the address of the proxy server to use to connect to HTTP services.

docs/en/setup/installing-xes.asciidoc

@@ -33,7 +33,7 @@ internet access:
 .. Manually download the {xpack} zip file:
 https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{version}.zip[
 +https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{version}.zip+]
-(https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{version}.zip.sha1[sha1])
+(https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{version}.zip.sha512[sha512])
 +
 --
 NOTE: The plugins for {es}, {kib}, and Logstash are included in the same zip

docs/en/setup/setup-xclient.asciidoc

@@ -2,14 +2,12 @@
 [[setup-xpack-client]]
 == Configuring {xpack} Java Clients
 
+deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.]
+
 If you want to use a Java {javaclient}/transport-client.html[transport client] with a
 cluster where {xpack} is installed, then you must download and configure the
 {xpack} transport client.
 
-WARNING: The `TransportClient` is aimed to be replaced by the Java High Level REST
-Client, which executes HTTP requests instead of serialized Java requests. The
-`TransportClient` will be deprecated in upcoming versions of {es}.
-
 . Add the {xpack} transport JAR file to your *CLASSPATH*. You can download the {xpack}
 distribution and extract the JAR file manually or you can get it from the
 https://artifacts.elastic.co/maven/org/elasticsearch/client/x-pack-transport/{version}/x-pack-transport-{version}.jar[Elasticsearc Maven repository].

docs/en/watcher/actions/pagerduty.asciidoc

@@ -45,7 +45,7 @@ payload as well as an array of contexts to the action.
       "attach_payload" : true,
       "client" : "/foo/bar/{{ctx.watch_id}}",
       "client_url" : "http://www.example.org/",
-      "context" : [
+      "contexts" : [
         {
           "type": "link",
           "href": "http://acme.pagerduty.com"

docs/en/watcher/java.asciidoc

@@ -1,6 +1,8 @@
 [[api-java]]
 == Java API
 
+deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.]
+
 {xpack} provides a Java client called `WatcherClient` that adds native Java
 support for the {watcher}.
 

docs/en/watcher/limitations.asciidoc

@@ -17,4 +17,12 @@ When you create a new watch or edit an existing watch, if you navigate away
 from the page without saving your changes they will be lost without warning. 
 Make sure to save your changes before leaving the page.
 
-image::watcher-ui-edit-watch.png[]
+image::watcher-ui-edit-watch.png[]
+
+[float]
+=== Security Integration
+
+When {security} is enabled, a watch stores information about what the user who
+stored the watch is allowed to execute **at that time**. This means, if those
+permissions change over time, the watch will still be able to execute with the
+permissions that existed when the watch was created.

docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java

@@ -7,7 +7,6 @@ package org.elasticsearch.smoketest;
 
 import org.apache.http.HttpHost;
 import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.SecureString;
@@ -17,9 +16,8 @@ import org.elasticsearch.test.rest.yaml.ClientYamlDocsTestClient;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestClient;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
-import org.elasticsearch.xpack.ml.integration.MlRestTestStateCleaner;
+import org.elasticsearch.xpack.test.rest.XPackRestIT;
 import org.junit.After;
 
 import java.io.IOException;
@@ -32,18 +30,13 @@ import static java.util.Collections.singletonMap;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.is;
 
-public class XDocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+public class XDocsClientYamlTestSuiteIT extends XPackRestIT {
     private static final String USER_TOKEN = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
 
     public XDocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
-        return ESClientYamlSuiteTestCase.createParameters();
-    }
-
     @Override
     protected void afterIfFailed(List<Throwable> errors) {
         super.afterIfFailed(errors);
@@ -93,11 +86,23 @@ public class XDocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         }
     }
 
-    private boolean isWatcherTest() {
+    @Override
+    protected boolean isWatcherTest() {
         String testName = getTestName();
         return testName != null && testName.contains("watcher");
     }
 
+    @Override
+    protected boolean isMonitoringTest() {
+        return false;
+    }
+
+    @Override
+    protected boolean isMachineLearningTest() {
+        String testName = getTestName();
+        return testName != null && testName.contains("ml/");
+    }
+
     /**
      * Deletes users after every test just in case any test adds any.
      */
@@ -117,11 +122,6 @@ public class XDocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         }
     }
 
-    @After
-    public void cleanMlState() throws Exception {
-        new MlRestTestStateCleaner(logger, adminClient(), this).clearMlMetadata();
-    }
-
     @Override
     protected boolean randomizeContentType() {
         return false;

plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java

@@ -47,6 +47,7 @@ import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -108,12 +109,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin {
+public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin,
+        DiscoveryPlugin, MapperPlugin {
 
     public static final String NAME = "x-pack";
 
@@ -565,4 +569,9 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
     public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
         return security.getJoinValidator();
     }
+
+    @Override
+    public Function<String, Predicate<String>> getFieldFilter() {
+        return security.getFieldFilter();
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/logstash/LogstashSecurityExtension.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.logstash;
+
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.SecurityExtension;
+import org.elasticsearch.xpack.security.support.MetadataUtils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class LogstashSecurityExtension implements SecurityExtension {
+    @Override
+    public Map<String, RoleDescriptor> getReservedRoles() {
+        Map<String, RoleDescriptor> roles = new HashMap<>();
+
+        roles.put("logstash_admin",
+                  new RoleDescriptor("logstash_admin",
+                                     null,
+                                     new RoleDescriptor.IndicesPrivileges[]{
+                                         RoleDescriptor.IndicesPrivileges.builder().indices(".logstash*")
+                                                 .privileges("create", "delete", "index", "manage", "read")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        return Collections.unmodifiableMap(roles);
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningSecurityExtension.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml;
+
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.SecurityExtension;
+import org.elasticsearch.xpack.security.support.MetadataUtils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class MachineLearningSecurityExtension implements SecurityExtension {
+    @Override
+    public Map<String, RoleDescriptor> getReservedRoles() {
+        Map<String, RoleDescriptor> roles = new HashMap<>();
+
+        roles.put("machine_learning_user",
+                  new RoleDescriptor("machine_learning_user",
+                                     new String[] { "monitor_ml" },
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".ml-anomalies*", ".ml-notifications")
+                                                 .privileges("view_index_metadata", "read")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        roles.put("machine_learning_admin",
+                  new RoleDescriptor("machine_learning_admin",
+                                     new String[] { "manage_ml" },
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".ml-*")
+                                                 .privileges("view_index_metadata", "read")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        return Collections.unmodifiableMap(roles);
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/ml/action/ForecastJobAction.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.config.Job;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams;
@@ -34,6 +35,8 @@ import org.elasticsearch.xpack.ml.job.results.Forecast;
 import java.io.IOException;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.ml.action.ForecastJobAction.Request.DURATION;
+
 public class ForecastJobAction extends Action<ForecastJobAction.Request, ForecastJobAction.Response, ForecastJobAction.RequestBuilder> {
 
     public static final ForecastJobAction INSTANCE = new ForecastJobAction();
@@ -244,13 +247,16 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
 
     public static class TransportAction extends TransportJobTaskAction<Request, Response> {
 
+        private final JobManager jobManager;
+
         @Inject
         public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ClusterService clusterService,
                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                AutodetectProcessManager processManager) {
+                JobManager jobManager, AutodetectProcessManager processManager) {
             super(settings, ForecastJobAction.NAME, threadPool, clusterService, transportService, actionFilters,
                     indexNameExpressionResolver, ForecastJobAction.Request::new, ForecastJobAction.Response::new, ThreadPool.Names.SAME,
                     processManager);
+            this.jobManager = jobManager;
             // ThreadPool.Names.SAME, because operations is executed by autodetect worker thread
         }
 
@@ -265,6 +271,16 @@ public class ForecastJobAction extends Action<ForecastJobAction.Request, Forecas
         protected void taskOperation(Request request, OpenJobAction.JobTask task, ActionListener<Response> listener) {
             ForecastParams.Builder paramsBuilder = ForecastParams.builder();
             if (request.getDuration() != null) {
+                TimeValue duration = request.getDuration();
+                TimeValue bucketSpan = jobManager.getJobOrThrowIfUnknown(task.getJobId()).getAnalysisConfig().getBucketSpan();
+
+                if (duration.compareTo(bucketSpan) < 0) {
+                    throw new IllegalArgumentException(
+                            "[" + DURATION.getPreferredName()
+                            + "] must be greater or equal to the bucket span: ["
+                            + duration.getStringRep() + "/" + bucketSpan.getStringRep() + "]");
+                }
+
                 paramsBuilder.duration(request.getDuration());
             }
             if (request.getExpiresIn() != null) {

plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java

@@ -49,6 +49,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -61,8 +62,10 @@ import org.elasticsearch.xpack.ml.MlMetadata;
 import org.elasticsearch.xpack.ml.job.config.Job;
 import org.elasticsearch.xpack.ml.job.config.JobState;
 import org.elasticsearch.xpack.ml.job.config.JobTaskStatus;
+import org.elasticsearch.xpack.ml.job.config.JobUpdate;
 import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings;
+import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.persistent.AllocatedPersistentTask;
@@ -390,15 +393,17 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
         private final XPackLicenseState licenseState;
         private final PersistentTasksService persistentTasksService;
         private final Client client;
+        private final JobProvider jobProvider;
 
         @Inject
         public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, XPackLicenseState licenseState,
                                ClusterService clusterService, PersistentTasksService persistentTasksService, ActionFilters actionFilters,
-                               IndexNameExpressionResolver indexNameExpressionResolver, Client client) {
+                               IndexNameExpressionResolver indexNameExpressionResolver, Client client, JobProvider jobProvider) {
             super(settings, NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new);
             this.licenseState = licenseState;
             this.persistentTasksService = persistentTasksService;
             this.client = client;
+            this.jobProvider = jobProvider;
         }
 
         @Override
@@ -422,10 +427,10 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
         }
 
         @Override
-        protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
+        protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
             JobParams jobParams = request.getJobParams();
             if (licenseState.isMachineLearningAllowed()) {
-                // Step 4. Wait for job to be started and respond
+                // Step 5. Wait for job to be started and respond
                 ActionListener<PersistentTask<JobParams>> finalListener = new ActionListener<PersistentTask<JobParams>>() {
                     @Override
                     public void onResponse(PersistentTask<JobParams> task) {
@@ -442,11 +447,42 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
                     }
                 };
 
-                // Step 3. Start job task
-                ActionListener<Boolean> missingMappingsListener = ActionListener.wrap(
+                // Step 4. Start job task
+                ActionListener<PutJobAction.Response> establishedMemoryUpdateListener = ActionListener.wrap(
                         response -> persistentTasksService.startPersistentTask(MlMetadata.jobTaskId(jobParams.jobId),
-                                TASK_NAME, jobParams, finalListener)
-                        , listener::onFailure
+                                TASK_NAME, jobParams, finalListener),
+                        listener::onFailure
+                );
+
+                // Step 3. Update established model memory for pre-6.1 jobs that haven't had it set
+                ActionListener<Boolean> missingMappingsListener = ActionListener.wrap(
+                        response -> {
+                            MlMetadata mlMetadata = clusterService.state().getMetaData().custom(MlMetadata.TYPE);
+                            Job job = mlMetadata.getJobs().get(jobParams.getJobId());
+                            if (job != null) {
+                                Version jobVersion = job.getJobVersion();
+                                Long jobEstablishedModelMemory = job.getEstablishedModelMemory();
+                                if ((jobVersion == null || jobVersion.before(Version.V_6_1_0))
+                                        && (jobEstablishedModelMemory == null || jobEstablishedModelMemory == 0)) {
+                                    jobProvider.getEstablishedMemoryUsage(job.getId(), null, null, establishedModelMemory -> {
+                                        if (establishedModelMemory != null && establishedModelMemory > 0) {
+                                            JobUpdate update = new JobUpdate.Builder(job.getId())
+                                                    .setEstablishedModelMemory(establishedModelMemory).build();
+                                            UpdateJobAction.Request updateRequest = new UpdateJobAction.Request(job.getId(), update);
+
+                                            executeAsyncWithOrigin(client, ML_ORIGIN, UpdateJobAction.INSTANCE, updateRequest,
+                                                    establishedMemoryUpdateListener);
+                                        } else {
+                                            establishedMemoryUpdateListener.onResponse(null);
+                                        }
+                                    }, listener::onFailure);
+                                } else {
+                                    establishedMemoryUpdateListener.onResponse(null);
+                                }
+                            } else {
+                                establishedMemoryUpdateListener.onResponse(null);
+                            }
+                        }, listener::onFailure
                 );
 
                 // Step 2. Try adding state doc mapping
@@ -502,7 +538,13 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
             String[] concreteIndices = aliasOrIndex.getIndices().stream().map(IndexMetaData::getIndex).map(Index::getName)
                     .toArray(String[]::new);
 
-            String[] indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, Version.CURRENT, logger);
+            String[] indicesThatRequireAnUpdate;
+            try {
+                indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, Version.CURRENT, logger);
+            } catch (IOException e) {
+                listener.onFailure(e);
+                return;
+            }
 
             if (indicesThatRequireAnUpdate.length > 0) {
                 try (XContentBuilder mapping = mappingSupplier.get()) {
@@ -707,7 +749,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
                 continue;
             }
 
-            if (nodeSupportsJobVersion(node.getVersion(), job.getJobVersion()) == false) {
+            if (nodeSupportsJobVersion(node.getVersion()) == false) {
                 String reason = "Not opening job [" + jobId + "] on node [" + node
                         + "], because this node does not support jobs of version [" + job.getJobVersion() + "]";
                 logger.trace(reason);
@@ -849,15 +891,16 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
         return unavailableIndices;
     }
 
-    static boolean nodeSupportsJobVersion(Version nodeVersion, Version jobVersion) {
+    private static boolean nodeSupportsJobVersion(Version nodeVersion) {
         return nodeVersion.onOrAfter(Version.V_5_5_0);
     }
 
-    static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion, Logger logger) {
+    static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion,
+                                          Logger logger) throws IOException {
         List<String> indicesToUpdate = new ArrayList<>();
 
         ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> currentMapping = state.metaData().findMappings(concreteIndices,
-                new String[] { ElasticsearchMappings.DOC_TYPE });
+                new String[] { ElasticsearchMappings.DOC_TYPE }, MapperPlugin.NOOP_FIELD_FILTER);
 
         for (String index : concreteIndices) {
             ImmutableOpenMap<String, MappingMetaData> innerMap = currentMapping.get(index);

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfig.java

@@ -54,6 +54,11 @@ import java.util.concurrent.TimeUnit;
  */
 public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements ToXContentObject {
 
+    private static final int SECONDS_IN_MINUTE = 60;
+    private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE;
+    private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE;
+    private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE;
+
     // Used for QueryPage
     public static final ParseField RESULTS_FIELD = new ParseField("datafeeds");
 
@@ -350,6 +355,53 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
         return Strings.toString(this);
     }
 
+    /**
+     * Calculates a sensible default frequency for a given bucket span.
+     * <p>
+     * The default depends on the bucket span:
+     * <ul>
+     * <li> &lt;= 2 mins -&gt; 1 min</li>
+     * <li> &lt;= 20 mins -&gt; bucket span / 2</li>
+     * <li> &lt;= 12 hours -&gt; 10 mins</li>
+     * <li> &gt; 12 hours -&gt; 1 hour</li>
+     * </ul>
+     *
+     * If the datafeed has aggregations, the default frequency is the
+     * closest multiple of the histogram interval based on the rules above.
+     *
+     * @param bucketSpan the bucket span
+     * @return the default frequency
+     */
+    public TimeValue defaultFrequency(TimeValue bucketSpan) {
+        TimeValue defaultFrequency = defaultFrequencyTarget(bucketSpan);
+        if (hasAggregations()) {
+            long histogramIntervalMillis = getHistogramIntervalMillis();
+            long targetFrequencyMillis = defaultFrequency.millis();
+            long defaultFrequencyMillis = histogramIntervalMillis > targetFrequencyMillis ? histogramIntervalMillis
+                    : (targetFrequencyMillis / histogramIntervalMillis) * histogramIntervalMillis;
+            defaultFrequency = TimeValue.timeValueMillis(defaultFrequencyMillis);
+        }
+        return defaultFrequency;
+    }
+
+    private TimeValue defaultFrequencyTarget(TimeValue bucketSpan) {
+        long bucketSpanSeconds = bucketSpan.seconds();
+        if (bucketSpanSeconds <= 0) {
+            throw new IllegalArgumentException("Bucket span has to be > 0");
+        }
+
+        if (bucketSpanSeconds <= TWO_MINS_SECONDS) {
+            return TimeValue.timeValueSeconds(SECONDS_IN_MINUTE);
+        }
+        if (bucketSpanSeconds <= TWENTY_MINS_SECONDS) {
+            return TimeValue.timeValueSeconds(bucketSpanSeconds / 2);
+        }
+        if (bucketSpanSeconds <= HALF_DAY_SECONDS) {
+            return TimeValue.timeValueMinutes(10);
+        }
+        return TimeValue.timeValueHours(1);
+    }
+
     public static class Builder {
 
         private static final int DEFAULT_SCROLL_SIZE = 1000;

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java

@@ -11,6 +11,7 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -96,8 +97,11 @@ class DatafeedJob {
 
         String msg = Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STARTED_FROM_TO,
                 DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(lookbackStartTimeMs),
-                endTime == null ? "real-time" : DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(lookbackEnd));
+                endTime == null ? "real-time" : DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(lookbackEnd),
+                TimeValue.timeValueMillis(frequencyMs).getStringRep());
         auditor.info(jobId, msg);
+        LOGGER.info("[{}] {}", jobId, msg);
+
 
         FlushJobAction.Request request = new FlushJobAction.Request(jobId);
         request.setCalcInterim(true);
@@ -114,7 +118,7 @@ class DatafeedJob {
             }
         }
         if (!isIsolated) {
-            LOGGER.debug("Lookback finished after being stopped");
+            LOGGER.debug("[{}] Lookback finished after being stopped", jobId);
         }
         return null;
     }
@@ -129,7 +133,7 @@ class DatafeedJob {
             FlushJobAction.Request request = new FlushJobAction.Request(jobId);
             request.setSkipTime(String.valueOf(startTime));
             FlushJobAction.Response flushResponse = flushJob(request);
-            LOGGER.info("Skipped to time [" + flushResponse.getLastFinalizedBucketEnd().getTime() + "]");
+            LOGGER.info("[{}] Skipped to time [{}]", jobId, flushResponse.getLastFinalizedBucketEnd().getTime());
             return flushResponse.getLastFinalizedBucketEnd().getTime();
         }
         return startTime;

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java

@@ -20,7 +20,6 @@ import org.elasticsearch.xpack.ml.job.results.Bucket;
 import org.elasticsearch.xpack.ml.job.results.Result;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 
-import java.time.Duration;
 import java.util.Collections;
 import java.util.Objects;
 import java.util.function.Consumer;
@@ -47,9 +46,9 @@ public class DatafeedJobBuilder {
 
         // Step 5. Build datafeed job object
         Consumer<Context> contextHanlder = context -> {
-            Duration frequency = getFrequencyOrDefault(datafeed, job);
-            Duration queryDelay = Duration.ofMillis(datafeed.getQueryDelay().millis());
-            DatafeedJob datafeedJob = new DatafeedJob(job.getId(), buildDataDescription(job), frequency.toMillis(), queryDelay.toMillis(),
+            TimeValue frequency = getFrequencyOrDefault(datafeed, job);
+            TimeValue queryDelay = datafeed.getQueryDelay();
+            DatafeedJob datafeedJob = new DatafeedJob(job.getId(), buildDataDescription(job), frequency.millis(), queryDelay.millis(),
                     context.dataExtractorFactory, client, auditor, currentTimeSupplier,
                     context.latestFinalBucketEndMs, context.latestRecordTimeMs);
             listener.onResponse(datafeedJob);
@@ -100,10 +99,13 @@ public class DatafeedJobBuilder {
         });
     }
 
-    private static Duration getFrequencyOrDefault(DatafeedConfig datafeed, Job job) {
+    private static TimeValue getFrequencyOrDefault(DatafeedConfig datafeed, Job job) {
         TimeValue frequency = datafeed.getFrequency();
-        TimeValue bucketSpan = job.getAnalysisConfig().getBucketSpan();
-        return frequency == null ? DefaultFrequency.ofBucketSpan(bucketSpan.seconds()) : Duration.ofSeconds(frequency.seconds());
+        if (frequency == null) {
+            TimeValue bucketSpan = job.getAnalysisConfig().getBucketSpan();
+            return datafeed.defaultFrequency(bucketSpan);
+        }
+        return frequency;
     }
 
     private static DataDescription buildDataDescription(Job job) {

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidator.java

@@ -29,6 +29,7 @@ public final class DatafeedJobValidator {
         if (datafeedConfig.hasAggregations()) {
             checkSummaryCountFieldNameIsSet(analysisConfig);
             checkValidHistogramInterval(datafeedConfig, analysisConfig);
+            checkFrequencyIsMultipleOfHistogramInterval(datafeedConfig);
         }
     }
 
@@ -55,6 +56,18 @@ public final class DatafeedJobValidator {
                     TimeValue.timeValueMillis(histogramIntervalMillis).getStringRep(),
                     TimeValue.timeValueMillis(bucketSpanMillis).getStringRep()));
         }
+    }
 
+    private static void checkFrequencyIsMultipleOfHistogramInterval(DatafeedConfig datafeedConfig) {
+        TimeValue frequency = datafeedConfig.getFrequency();
+        if (frequency != null) {
+            long histogramIntervalMillis = datafeedConfig.getHistogramIntervalMillis();
+            long frequencyMillis = frequency.millis();
+            if (frequencyMillis % histogramIntervalMillis != 0) {
+                throw ExceptionsHelper.badRequestException(Messages.getMessage(
+                        Messages.DATAFEED_FREQUENCY_MUST_BE_MULTIPLE_OF_AGGREGATIONS_INTERVAL,
+                        frequency, TimeValue.timeValueMillis(histogramIntervalMillis).getStringRep()));
+            }
+        }
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.datafeed;
 
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -16,7 +17,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.MlMetadata;
@@ -49,8 +50,6 @@ import static org.elasticsearch.xpack.persistent.PersistentTasksService.WaitForP
 
 public class DatafeedManager extends AbstractComponent {
 
-    private static final String INF_SYMBOL = "forever";
-
     private final Client client;
     private final ClusterService clusterService;
     private final PersistentTasksService persistentTasksService;
@@ -157,9 +156,6 @@ public class DatafeedManager extends AbstractComponent {
     // otherwise if a stop datafeed call is made immediately after the start datafeed call we could cancel
     // the DatafeedTask without stopping datafeed, which causes the datafeed to keep on running.
     private void innerRun(Holder holder, long startTime, Long endTime) {
-        logger.info("Starting datafeed [{}] for job [{}] in [{}, {})", holder.datafeed.getId(), holder.datafeed.getJobId(),
-                DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(startTime),
-                endTime == null ? INF_SYMBOL : DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(endTime));
         holder.future = threadPool.executor(MachineLearning.DATAFEED_THREAD_POOL_NAME).submit(new AbstractRunnable() {
 
             @Override
@@ -429,7 +425,15 @@ public class DatafeedManager extends AbstractComponent {
 
                                 @Override
                                 public void onFailure(Exception e) {
-                                    logger.error("[" + getJobId() + "] failed to auto-close job", e);
+                                    // Given that the UI force-deletes the datafeed and then force-deletes the job, it's
+                                    // quite likely that the auto-close here will get interrupted by a process kill request,
+                                    // and it's misleading/worrying to log an error in this case.
+                                    if (e instanceof ElasticsearchStatusException &&
+                                            ((ElasticsearchStatusException) e).status() == RestStatus.CONFLICT) {
+                                        logger.debug("[{}] {}", getJobId(), e.getMessage());
+                                    } else {
+                                        logger.error("[" + getJobId() + "] failed to auto-close job", e);
+                                    }
                                 }
                             });
                 }

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/DefaultFrequency.java

@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.datafeed;
-
-import java.time.Duration;
-
-/**
- * Factory methods for a sensible default for the datafeed frequency
- */
-public final class DefaultFrequency {
-    private static final int SECONDS_IN_MINUTE = 60;
-    private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE;
-    private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE;
-    private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE;
-    private static final Duration TEN_MINUTES = Duration.ofMinutes(10);
-    private static final Duration ONE_HOUR = Duration.ofHours(1);
-
-    private DefaultFrequency() {
-        // Do nothing
-    }
-
-    /**
-     * Creates a sensible default frequency for a given bucket span.
-     * <p>
-     * The default depends on the bucket span:
-     * <ul>
-     * <li> &lt;= 2 mins -&gt; 1 min</li>
-     * <li> &lt;= 20 mins -&gt; bucket span / 2</li>
-     * <li> &lt;= 12 hours -&gt; 10 mins</li>
-     * <li> &gt; 12 hours -&gt; 1 hour</li>
-     * </ul>
-     *
-     * @param bucketSpanSeconds the bucket span in seconds
-     * @return the default frequency
-     */
-    public static Duration ofBucketSpan(long bucketSpanSeconds) {
-        if (bucketSpanSeconds <= 0) {
-            throw new IllegalArgumentException("Bucket span has to be > 0");
-        }
-
-        if (bucketSpanSeconds <= TWO_MINS_SECONDS) {
-            return Duration.ofSeconds(SECONDS_IN_MINUTE);
-        }
-        if (bucketSpanSeconds <= TWENTY_MINS_SECONDS) {
-            return Duration.ofSeconds(bucketSpanSeconds / 2);
-        }
-        if (bucketSpanSeconds <= HALF_DAY_SECONDS) {
-            return TEN_MINUTES;
-        }
-        return ONE_HOUR;
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java

@@ -98,8 +98,8 @@ public class ChunkedDataExtractor implements DataExtractor {
             currentEnd = currentStart;
             chunkSpan = context.chunkSpan == null ? dataSummary.estimateChunk() : context.chunkSpan.getMillis();
             chunkSpan = context.timeAligner.alignToCeil(chunkSpan);
-            LOGGER.debug("Chunked search configured:  totalHits = {}, dataTimeSpread = {} ms, chunk span = {} ms",
-                    dataSummary.totalHits, dataSummary.getDataTimeSpread(), chunkSpan);
+            LOGGER.debug("[{}]Chunked search configured:  totalHits = {}, dataTimeSpread = {} ms, chunk span = {} ms",
+                    context.jobId, dataSummary.totalHits, dataSummary.getDataTimeSpread(), chunkSpan);
         } else {
             // search is over
             currentEnd = context.end;
@@ -164,7 +164,7 @@ public class ChunkedDataExtractor implements DataExtractor {
         currentStart = currentEnd;
         currentEnd = Math.min(currentStart + chunkSpan, context.end);
         currentExtractor = dataExtractorFactory.newExtractor(currentStart, currentEnd);
-        LOGGER.trace("advances time to [{}, {})", currentStart, currentEnd);
+        LOGGER.trace("[{}] advances time to [{}, {})", context.jobId, currentStart, currentEnd);
     }
 
     @Override

plugin/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java

@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ml.datafeed.extractor.scroll;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
@@ -214,13 +215,15 @@ class ScrollDataExtractor implements DataExtractor {
     }
 
     private void resetScroll() {
-        if (scrollId != null) {
-            clearScroll(scrollId);
-        }
+        clearScroll(scrollId);
         scrollId = null;
     }
 
-    void clearScroll(String scrollId) {
-        ClearScrollAction.INSTANCE.newRequestBuilder(client).addScrollId(scrollId).get();
+    private void clearScroll(String scrollId) {
+        if (scrollId != null) {
+            ClearScrollRequest request = new ClearScrollRequest();
+            request.addScrollId(scrollId);
+            client.execute(ClearScrollAction.INSTANCE, request).actionGet();
+        }
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java

@@ -39,6 +39,8 @@ public final class Messages {
     public static final String DATAFEED_DATA_HISTOGRAM_MUST_HAVE_NESTED_MAX_AGGREGATION =
             "Date histogram must have nested max aggregation for time_field [{0}]";
     public static final String DATAFEED_MISSING_MAX_AGGREGATION_FOR_TIME_FIELD = "Missing max aggregation for time_field [{0}]";
+    public static final String DATAFEED_FREQUENCY_MUST_BE_MULTIPLE_OF_AGGREGATIONS_INTERVAL =
+            "Datafeed frequency [{0}] must be a multiple of the aggregation interval [{1}]";
 
     public static final String INCONSISTENT_ID =
             "Inconsistent {0}; ''{1}'' specified in the body differs from ''{2}'' specified as a URL argument";
@@ -58,7 +60,7 @@ public final class Messages {
     public static final String JOB_AUDIT_DATAFEED_LOOKBACK_NO_DATA = "Datafeed lookback retrieved no data";
     public static final String JOB_AUDIT_DATAFEED_NO_DATA = "Datafeed has been retrieving no data for a while";
     public static final String JOB_AUDIT_DATAFEED_RECOVERED = "Datafeed has recovered data extraction and analysis";
-    public static final String JOB_AUDIT_DATAFEED_STARTED_FROM_TO = "Datafeed started (from: {0} to: {1})";
+    public static final String JOB_AUDIT_DATAFEED_STARTED_FROM_TO = "Datafeed started (from: {0} to: {1}) with frequency [{2}]";
     public static final String JOB_AUDIT_DATAFEED_STARTED_REALTIME = "Datafeed started in real-time";
     public static final String JOB_AUDIT_DATAFEED_STOPPED = "Datafeed stopped";
     public static final String JOB_AUDIT_DELETED = "Job deleted";

plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.persistence;
 
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -17,6 +18,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
@@ -191,22 +193,14 @@ public class JobStorageDeletionTask extends Task {
         executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, aliasesRequest,
                 ActionListener.<GetAliasesResponse>wrap(
                         getAliasesResponse -> {
-                            Set<String> aliases = new HashSet<>();
-                            getAliasesResponse.getAliases().valuesIt().forEachRemaining(
-                                    metaDataList -> metaDataList.forEach(metadata -> aliases.add(metadata.getAlias())));
-                            if (aliases.isEmpty()) {
+                            // remove the aliases from the concrete indices found in the first step
+                            IndicesAliasesRequest removeRequest = buildRemoveAliasesRequest(getAliasesResponse);
+                            if (removeRequest == null) {
                                 // don't error if the job's aliases have already been deleted - carry on and delete the
                                 // rest of the job's data
                                 finishedHandler.onResponse(true);
                                 return;
                             }
-                            List<String> indices = new ArrayList<>();
-                            getAliasesResponse.getAliases().keysIt().forEachRemaining(indices::add);
-                            // remove the aliases from the concrete indices found in the first step
-                            IndicesAliasesRequest removeRequest = new IndicesAliasesRequest().addAliasAction(
-                                    IndicesAliasesRequest.AliasActions.remove()
-                                            .aliases(aliases.toArray(new String[aliases.size()]))
-                                            .indices(indices.toArray(new String[indices.size()])));
                             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, removeRequest,
                                     ActionListener.<IndicesAliasesResponse>wrap(removeResponse -> finishedHandler.onResponse(true),
                                             finishedHandler::onFailure),
@@ -214,4 +208,21 @@ public class JobStorageDeletionTask extends Task {
                         },
                         finishedHandler::onFailure), client.admin().indices()::getAliases);
     }
+
+    private IndicesAliasesRequest buildRemoveAliasesRequest(GetAliasesResponse getAliasesResponse) {
+        Set<String> aliases = new HashSet<>();
+        List<String> indices = new ArrayList<>();
+        for (ObjectObjectCursor<String, List<AliasMetaData>> entry : getAliasesResponse.getAliases()) {
+            // The response includes _all_ indices, but only those associated with
+            // the aliases we asked about will have associated AliasMetaData
+            if (entry.value.isEmpty() == false) {
+                indices.add(entry.key);
+                entry.value.forEach(metadata -> aliases.add(metadata.getAlias()));
+            }
+        }
+        return aliases.isEmpty() ? null : new IndicesAliasesRequest().addAliasAction(
+                IndicesAliasesRequest.AliasActions.remove()
+                        .aliases(aliases.toArray(new String[aliases.size()]))
+                        .indices(indices.toArray(new String[indices.size()])));
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java

@@ -8,7 +8,6 @@ package org.elasticsearch.xpack.ml.job.process.autodetect;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.Loggers;
@@ -32,6 +31,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.ml.job.process.autodetect.writer.DataToProcessWriter;
 import org.elasticsearch.xpack.ml.job.process.autodetect.writer.DataToProcessWriterFactory;
+import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -154,7 +154,12 @@ public class AutodetectCommunicator implements Closeable {
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
         } catch (ExecutionException e) {
-            throw ExceptionsHelper.convertToElastic(e);
+            if (processKilled) {
+                // In this case the original exception is spurious and highly misleading
+                throw ExceptionsHelper.conflictStatusException("Close job interrupted by kill request");
+            } else {
+                throw new ElasticsearchException(e);
+            }
         }
     }
 
@@ -242,19 +247,15 @@ public class AutodetectCommunicator implements Closeable {
      */
     private void checkProcessIsAlive() {
         if (!autodetectProcess.isProcessAlive()) {
-            ParameterizedMessage message =
-                    new ParameterizedMessage("[{}] Unexpected death of autodetect: {}", job.getId(), autodetectProcess.readError());
-            LOGGER.error(message);
-            throw new ElasticsearchException(message.getFormattedMessage());
+            // Don't log here - it just causes double logging when the exception gets logged
+            throw new ElasticsearchException("[{}] Unexpected death of autodetect: {}", job.getId(), autodetectProcess.readError());
         }
     }
 
     private void checkResultsProcessorIsAlive() {
         if (autoDetectResultProcessor.isFailed()) {
-            ParameterizedMessage message =
-                    new ParameterizedMessage("[{}] Unexpected death of the result processor", job.getId());
-            LOGGER.error(message);
-            throw new ElasticsearchException(message.getFormattedMessage());
+            // Don't log here - it just causes double logging when the exception gets logged
+            throw new ElasticsearchException("[{}] Unexpected death of the result processor", job.getId());
         }
     }
 

plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java

@@ -88,7 +88,7 @@ public class AutodetectProcessManager extends AbstractComponent {
     // TODO: Remove the deprecated setting in 7.0 and move the default value to the replacement setting
     @Deprecated
     public static final Setting<Integer> MAX_RUNNING_JOBS_PER_NODE =
-            Setting.intSetting("max_running_jobs", 10, 1, 512, Property.NodeScope, Property.Deprecated);
+            Setting.intSetting("max_running_jobs", 20, 1, 512, Property.NodeScope, Property.Deprecated);
     public static final Setting<Integer> MAX_OPEN_JOBS_PER_NODE =
             Setting.intSetting("xpack.ml.max_open_jobs", MAX_RUNNING_JOBS_PER_NODE, 1, Property.NodeScope);
 
@@ -473,6 +473,10 @@ public class AutodetectProcessManager extends AbstractComponent {
             communicator.close(restart, reason);
             processByAllocation.remove(allocationId);
         } catch (Exception e) {
+            // If the close failed because the process has explicitly been killed by us then just pass on that exception
+            if (e instanceof ElasticsearchStatusException && ((ElasticsearchStatusException) e).status() == RestStatus.CONFLICT) {
+                throw e;
+            }
             logger.warn("[" + jobId + "] Exception closing autodetect process", e);
             setJobState(jobTask, JobState.FAILED);
             throw ExceptionsHelper.serverError("Exception closing autodetect process", e);

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java

@@ -13,8 +13,8 @@ public enum MonitoredSystem {
 
     ES("es"),
     KIBANA("kibana"),
-    // TODO: when "BEATS" is re-added, add it to tests where we randomly select "LOGSTASH"
     LOGSTASH("logstash"),
+    BEATS("beats"),
     UNKNOWN("unknown");
 
     private final String system;
@@ -35,6 +35,8 @@ public enum MonitoredSystem {
                 return KIBANA;
             case "logstash":
                 return LOGSTASH;
+            case "beats":
+                return BEATS;
             default:
                 // Return an "unknown" monitored system
                 // that can easily be filtered out if

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSecurityExtension.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring;
+
+import org.elasticsearch.xpack.monitoring.action.MonitoringBulkAction;
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.SecurityExtension;
+import org.elasticsearch.xpack.security.support.MetadataUtils;
+import org.elasticsearch.xpack.security.user.KibanaUser;
+import org.elasticsearch.xpack.security.user.LogstashSystemUser;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class MonitoringSecurityExtension implements SecurityExtension {
+    @Override
+    public Map<String, RoleDescriptor> getReservedRoles() {
+        Map<String, RoleDescriptor> roles = new HashMap<>();
+
+        roles.put("monitoring_user",
+                  new RoleDescriptor("monitoring_user",
+                                     null,
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".monitoring-*")
+                                                 .privileges("read", "read_cross_cluster")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        roles.put("remote_monitoring_agent",
+                  new RoleDescriptor("remote_monitoring_agent",
+                                     new String[] {
+                                          "manage_index_templates", "manage_ingest_pipelines", "monitor",
+                                          "cluster:monitor/xpack/watcher/watch/get",
+                                          "cluster:admin/xpack/watcher/watch/put",
+                                          "cluster:admin/xpack/watcher/watch/delete",
+                                     },
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".monitoring-*")
+                                                 .privileges("all")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        // TODO(core-infra) put KibanaUser & LogstashSystemUser into a common place for the split and use them here
+        roles.put("logstash_system",
+                  new RoleDescriptor(LogstashSystemUser.ROLE_NAME,
+                                     new String[]{"monitor", MonitoringBulkAction.NAME},
+                                     null,
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        roles.put("kibana_system",
+                new RoleDescriptor(KibanaUser.ROLE_NAME,
+                                     new String[] { "monitor", "manage_index_templates", MonitoringBulkAction.NAME },
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".kibana*", ".reporting-*")
+                                                 .privileges("all")
+                                                 .build(),
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".monitoring-*")
+                                                 .privileges("read", "read_cross_cluster")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        return Collections.unmodifiableMap(roles);
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtils.java

@@ -41,7 +41,7 @@ public final class MonitoringTemplateUtils {
     /**
      * IDs of templates that can be used with {@linkplain #loadTemplate(String) loadTemplate}.
      */
-    public static final String[] TEMPLATE_IDS = { "alerts", "es", "kibana", "logstash" };
+    public static final String[] TEMPLATE_IDS = { "alerts", "es", "kibana", "logstash", "beats" };
 
     /**
      * IDs of templates that can be used with {@linkplain #createEmptyTemplate(String) createEmptyTemplate} that are not managed by a

plugin/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java

@@ -56,7 +56,8 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
         final Map<MonitoredSystem, List<String>> versionsMap = new HashMap<>();
         versionsMap.put(MonitoredSystem.KIBANA, allVersions);
         versionsMap.put(MonitoredSystem.LOGSTASH, allVersions);
-        // Beats did not report data in the 5.x timeline, so it should never send the original version [when we add it!]
+        // Beats did not report data in the 5.x timeline, so it should never send the original version
+        versionsMap.put(MonitoredSystem.BEATS, Collections.singletonList(MonitoringTemplateUtils.TEMPLATE_VERSION));
         supportedApiVersions = Collections.unmodifiableMap(versionsMap);
     }
 

plugin/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -61,6 +61,7 @@ import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
@@ -139,9 +140,11 @@ import org.elasticsearch.xpack.security.authc.support.mapper.expressiondsl.Expre
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener;
+import org.elasticsearch.xpack.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache;
 import org.elasticsearch.xpack.security.authz.accesscontrol.SecurityIndexSearcherWrapper;
 import org.elasticsearch.xpack.security.authz.accesscontrol.SetSecurityUserProcessor;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
@@ -180,7 +183,6 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.security.GeneralSecurityException;
 import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -194,6 +196,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
@@ -206,7 +209,7 @@ import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY
 import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_TEMPLATE_NAME;
 import static org.elasticsearch.xpack.security.support.IndexLifecycleManager.INTERNAL_INDEX_FORMAT;
 
-public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin {
+public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin, MapperPlugin {
 
     private static final Logger logger = Loggers.getLogger(XPackPlugin.class);
 
@@ -239,8 +242,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
     private final SetOnce<SecurityActionFilter> securityActionFilter = new SetOnce<>();
     private final List<BootstrapCheck> bootstrapChecks;
 
-    public Security(Settings settings, Environment env, XPackLicenseState licenseState, SSLService sslService)
-                    throws IOException, GeneralSecurityException {
+    public Security(Settings settings, Environment env, XPackLicenseState licenseState, SSLService sslService) {
         this.settings = settings;
         this.env = env;
         this.transportClientMode = XPackPlugin.transportClientMode(settings);
@@ -343,7 +345,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
             }
         }
         final AuditTrailService auditTrailService =
-                new AuditTrailService(settings,  auditTrails.stream().collect(Collectors.toList()), licenseState);
+                new AuditTrailService(settings, new ArrayList<>(auditTrails), licenseState);
         components.add(auditTrailService);
         this.auditTrailService.set(auditTrailService);
 
@@ -359,9 +361,8 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         final ReservedRealm reservedRealm = new ReservedRealm(env, settings, nativeUsersStore,
                 anonymousUser, securityLifecycleService, threadPool.getThreadContext());
-        Map<String, Realm.Factory> realmFactories = new HashMap<>();
-        realmFactories.putAll(InternalRealms.getFactories(threadPool, resourceWatcherService, sslService, nativeUsersStore,
-                nativeRoleMappingStore, securityLifecycleService));
+        Map<String, Realm.Factory> realmFactories = new HashMap<>(InternalRealms.getFactories(threadPool, resourceWatcherService,
+                sslService, nativeUsersStore, nativeRoleMappingStore, securityLifecycleService));
         for (XPackExtension extension : extensions) {
             Map<String, Realm.Factory> newRealms = extension.getRealms(resourceWatcherService);
             for (Map.Entry<String, Realm.Factory> entry : newRealms.entrySet()) {
@@ -529,11 +530,8 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
 
     
     public List<String> getSettingsFilter(@Nullable XPackExtensionsService extensionsService) {
-        ArrayList<String> settingsFilter = new ArrayList<>();
         List<String> asArray = settings.getAsList(setting("hide_settings"));
-        for (String pattern : asArray) {
-            settingsFilter.add(pattern);
-        }
+        ArrayList<String> settingsFilter = new ArrayList<>(asArray);
 
         final List<XPackExtension> extensions = extensionsService == null ? Collections.emptyList() : extensionsService.getExtensions();
         settingsFilter.addAll(RealmSettings.getSettingsFilter(extensions));
@@ -775,8 +773,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
         }
 
         String[] matches = Strings.commaDelimitedListToStringArray(value);
-        List<String> indices = new ArrayList<>();
-        indices.addAll(SecurityLifecycleService.indexNames());
+        List<String> indices = new ArrayList<>(SecurityLifecycleService.indexNames());
         if (indexAuditingEnabled) {
             DateTime now = new DateTime(DateTimeZone.UTC);
             // just use daily rollover
@@ -941,6 +938,31 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
         }
     }
 
+    @Override
+    public Function<String, Predicate<String>> getFieldFilter() {
+        if (enabled) {
+            return index -> {
+                if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) {
+                    return MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+                IndicesAccessControl indicesAccessControl = threadContext.get().getTransient(AuthorizationService.INDICES_PERMISSIONS_KEY);
+                IndicesAccessControl.IndexAccessControl indexPermissions = indicesAccessControl.getIndexPermissions(index);
+                if (indexPermissions == null) {
+                    return MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+                if (indexPermissions.isGranted() == false) {
+                    throw new IllegalStateException("unexpected call to getFieldFilter for index [" + index + "] which is not granted");
+                }
+                FieldPermissions fieldPermissions = indexPermissions.getFieldPermissions();
+                if (fieldPermissions == null) {
+                    return MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+                return fieldPermissions::grantsAccessTo;
+            };
+        }
+        return MapperPlugin.super.getFieldFilter();
+    }
+
     @Override
     public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
         if (enabled) {

plugin/src/main/java/org/elasticsearch/xpack/security/SecurityExtension.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * SPI interface to any plugins that want to provide custom extensions to aid the security module in functioning without
+ * needing to explicitly know about the behavior of the implementing plugin.
+ */
+public interface SecurityExtension {
+
+    /**
+     * Gets a set of reserved roles, consisting of the role name and the descriptor.
+     */
+    default Map<String, RoleDescriptor> getReservedRoles() {
+        return Collections.emptyMap();
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/StackSecurityExtension.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.support.MetadataUtils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class StackSecurityExtension implements SecurityExtension {
+    @Override
+    public Map<String, RoleDescriptor> getReservedRoles() {
+        Map<String, RoleDescriptor> roles = new HashMap<>();
+
+        roles.put("transport_client",
+                  new RoleDescriptor("transport_client",
+                                     new String[] { "transport_client" },
+                                     null,
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        roles.put("kibana_user",
+                  new RoleDescriptor("kibana_user",
+                                     null,
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".kibana*")
+                                                 .privileges("manage", "read", "index", "delete")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        roles.put("ingest_admin",
+                  new RoleDescriptor("ingest_admin",
+                                     new String[] { "manage_index_templates", "manage_pipeline" },
+                                     null,
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        // reporting_user doesn't have any privileges in Elasticsearch, and Kibana authorizes privileges based on this role
+        roles.put("reporting_user",
+                  new RoleDescriptor("reporting_user",
+                                     null,
+                                     null,
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        roles.put("kibana_dashboard_only_user",
+                  new RoleDescriptor("kibana_dashboard_only_user",
+                                     null,
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(".kibana*")
+                                                 .privileges("read", "view_index_metadata")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        return Collections.unmodifiableMap(roles);
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java

@@ -6,19 +6,20 @@
 package org.elasticsearch.xpack.security.action.user;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -68,10 +69,9 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
 
     private void checkPrivileges(HasPrivilegesRequest request, Role userRole,
                                  ActionListener<HasPrivilegesResponse> listener) {
-        if (logger.isDebugEnabled()) {
-            logger.debug("Check whether role [{}] has privileges cluster=[{}] index=[{}]", userRole.name(),
-                    Arrays.toString(request.clusterPrivileges()), Arrays.toString(request.indexPrivileges()));
-        }
+        logger.debug(() -> new ParameterizedMessage("Check whether role [{}] has privileges cluster=[{}] index=[{}]",
+                Strings.arrayToCommaDelimitedString(userRole.names()), Strings.arrayToCommaDelimitedString(request.clusterPrivileges()),
+                Strings.arrayToCommaDelimitedString(request.indexPrivileges())));
 
         Map<String, Boolean> cluster = new HashMap<>();
         for (String checkAction : request.clusterPrivileges()) {
@@ -93,10 +93,12 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
                 }
                 for (String privilege : check.getPrivileges()) {
                     if (testIndexMatch(index, privilege, userRole, predicateCache)) {
-                        logger.debug("Role [{}] has [{}] on [{}]", userRole.name(), privilege, index);
+                        logger.debug(() -> new ParameterizedMessage("Role [{}] has [{}] on [{}]",
+                                Strings.arrayToCommaDelimitedString(userRole.names()), privilege, index));
                         privileges.put(privilege, true);
                     } else {
-                        logger.debug("Role [{}] does not have [{}] on [{}]", userRole.name(), privilege, index);
+                        logger.debug(() -> new ParameterizedMessage("Role [{}] does not have [{}] on [{}]",
+                                Strings.arrayToCommaDelimitedString(userRole.names()), privilege, index));
                         privileges.put(privilege, false);
                         allMatch = false;
                     }

plugin/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java

@@ -39,25 +39,9 @@ public interface AuditTrail {
 
     void authenticationFailed(String realm, AuthenticationToken token, RestRequest request);
 
-    /**
-     * Access was granted for some request.
-     * @param specificIndices if non-null then the action was authorized
-     *      for all indices in this particular set of indices, otherwise
-     *      the action was authorized for all indices to which it is
-     *      related, if any
-     */
-    void accessGranted(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices);
+    void accessGranted(User user, String action, TransportMessage message, String[] roleNames, @Nullable Set<String> specificIndices);
 
-    /**
-     * Access was denied for some request.
-     * @param specificIndices if non-null then the action was denied
-     *      for at least one index in this particular set of indices,
-     *      otherwise the action was denied for at least one index
-     *      to which the request is related. If the request isn't
-     *      related to any particular index then the request itself
-     *      was denied.
-     */
-    void accessDenied(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices);
+    void accessDenied(User user, String action, TransportMessage message, String[] roleNames, @Nullable Set<String> specificIndices);
 
     void tamperedRequest(RestRequest request);
 
@@ -69,9 +53,9 @@ public interface AuditTrail {
 
     void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule);
 
-    void runAsGranted(User user, String action, TransportMessage message);
+    void runAsGranted(User user, String action, TransportMessage message, String[] roleNames);
 
-    void runAsDenied(User user, String action, TransportMessage message);
+    void runAsDenied(User user, String action, TransportMessage message, String[] roleNames);
 
-    void runAsDenied(User user, RestRequest request);
+    void runAsDenied(User user, RestRequest request, String[] roleNames);
 }

plugin/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java

@@ -132,19 +132,19 @@ public class AuditTrailService extends AbstractComponent implements AuditTrail {
     }
 
     @Override
-    public void accessGranted(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices) {
+    public void accessGranted(User user, String action, TransportMessage message, String[] roleNames,  @Nullable Set<String> specificIndices) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.accessGranted(user, action, message, specificIndices);
+                auditTrail.accessGranted(user, action, message, roleNames, specificIndices);
             }
         }
     }
 
     @Override
-    public void accessDenied(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices) {
+    public void accessDenied(User user, String action, TransportMessage message, String[] roleNames,  @Nullable Set<String> specificIndices) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.accessDenied(user, action, message, specificIndices);
+                auditTrail.accessDenied(user, action, message, roleNames, specificIndices);
             }
         }
     }
@@ -193,28 +193,28 @@ public class AuditTrailService extends AbstractComponent implements AuditTrail {
     }
 
     @Override
-    public void runAsGranted(User user, String action, TransportMessage message) {
+    public void runAsGranted(User user, String action, TransportMessage message, String[] roleNames) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.runAsGranted(user, action, message);
+                auditTrail.runAsGranted(user, action, message, roleNames);
             }
         }
     }
 
     @Override
-    public void runAsDenied(User user, String action, TransportMessage message) {
+    public void runAsDenied(User user, String action, TransportMessage message, String[] roleNames) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.runAsDenied(user, action, message);
+                auditTrail.runAsDenied(user, action, message, roleNames);
             }
         }
     }
 
     @Override
-    public void runAsDenied(User user, RestRequest request) {
+    public void runAsDenied(User user, RestRequest request, String[] roleNames) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.runAsDenied(user, request);
+                auditTrail.runAsDenied(user, request, roleNames);
             }
         }
     }

plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java

@@ -48,7 +48,6 @@ import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.authc.AuthenticationToken;
-import org.elasticsearch.xpack.security.authz.privilege.SystemPrivilege;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
 import org.elasticsearch.xpack.security.support.IndexLifecycleManager;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
@@ -356,7 +355,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationSuccess(String realm, User user, RestRequest request) {
         if (events.contains(AUTHENTICATION_SUCCESS)) {
             try {
-                enqueue(message("authentication_success", realm, user, request), "authentication_success");
+                enqueue(message("authentication_success", realm, user, null, request), "authentication_success");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [authentication_success]", e);
             }
@@ -367,7 +366,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationSuccess(String realm, User user, String action, TransportMessage message) {
         if (events.contains(AUTHENTICATION_SUCCESS)) {
             try {
-                enqueue(message("authentication_success", action, user, realm, null, message), "authentication_success");
+                enqueue(message("authentication_success", action, user, null, realm, null, message), "authentication_success");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [authentication_success]", e);
             }
@@ -378,7 +377,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     public void anonymousAccessDenied(String action, TransportMessage message) {
         if (events.contains(ANONYMOUS_ACCESS_DENIED)) {
             try {
-                enqueue(message("anonymous_access_denied", action, (User) null, null, indices(message), message),
+                enqueue(message("anonymous_access_denied", action, (User) null, null, null, indices(message), message),
                         "anonymous_access_denied");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [anonymous_access_denied]", e);
@@ -401,7 +400,8 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationFailed(String action, TransportMessage message) {
         if (events.contains(AUTHENTICATION_FAILED)) {
             try {
-                enqueue(message("authentication_failed", action, (User) null, null, indices(message), message), "authentication_failed");
+                enqueue(message("authentication_failed", action, (User) null, null, null, indices(message), message),
+                        "authentication_failed");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [authentication_failed]", e);
             }
@@ -473,21 +473,15 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     }
 
     @Override
-    public void accessGranted(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices) {
-        // special treatment for internal system actions - only log if explicitly told to
-        if ((SystemUser.is(user) && SystemPrivilege.INSTANCE.predicate().test(action))) {
-            if (events.contains(SYSTEM_ACCESS_GRANTED)) {
-                try {
-                    Set<String> indices = specificIndices == null ? indices(message) : specificIndices;
-                    enqueue(message("access_granted", action, user, null, indices, message), "access_granted");
-                } catch (Exception e) {
-                    logger.warn("failed to index audit event: [access_granted]", e);
-                }
-            }
-        } else if (events.contains(ACCESS_GRANTED) && XPackUser.is(user) == false) {
+    public void accessGranted(User user, String action, TransportMessage message, String[] roleNames,
+                              @Nullable Set<String> specificIndices) {
+        final boolean isSystem = SystemUser.is(user) || XPackUser.is(user);
+        final boolean logSystemAccessGranted = isSystem && events.contains(SYSTEM_ACCESS_GRANTED);
+        final boolean shouldLog = logSystemAccessGranted || (isSystem == false && events.contains(ACCESS_GRANTED));
+        if (shouldLog) {
             try {
                 Set<String> indices = specificIndices == null ? indices(message) : specificIndices;
-                enqueue(message("access_granted", action, user, null, indices, message), "access_granted");
+                enqueue(message("access_granted", action, user, roleNames, null, indices, message), "access_granted");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [access_granted]", e);
             }
@@ -495,11 +489,12 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     }
 
     @Override
-    public void accessDenied(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices) {
+    public void accessDenied(User user, String action, TransportMessage message, String[] roleNames,
+                        @Nullable Set<String> specificIndices) {
         if (events.contains(ACCESS_DENIED) && XPackUser.is(user) == false) {
             try {
-                Set<String> indices = specificIndices == null ? indices(message) : specificIndices; 
-                enqueue(message("access_denied", action, user, null, indices, message), "access_denied");
+                Set<String> indices = specificIndices == null ? indices(message) : specificIndices;
+                enqueue(message("access_denied", action, user, roleNames, null, indices, message), "access_denied");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [access_denied]", e);
             }
@@ -521,7 +516,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     public void tamperedRequest(String action, TransportMessage message) {
         if (events.contains(TAMPERED_REQUEST)) {
             try {
-                enqueue(message("tampered_request", action, (User) null, null, indices(message), message), "tampered_request");
+                enqueue(message("tampered_request", action, (User) null, null, null, indices(message), message), "tampered_request");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [tampered_request]", e);
             }
@@ -532,7 +527,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     public void tamperedRequest(User user, String action, TransportMessage request) {
         if (events.contains(TAMPERED_REQUEST) && XPackUser.is(user) == false) {
             try {
-                enqueue(message("tampered_request", action, user, null, indices(request), request), "tampered_request");
+                enqueue(message("tampered_request", action, user, null, null, indices(request), request), "tampered_request");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [tampered_request]", e);
             }
@@ -562,10 +557,10 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     }
 
     @Override
-    public void runAsGranted(User user, String action, TransportMessage message) {
+    public void runAsGranted(User user, String action, TransportMessage message, String[] roleNames) {
         if (events.contains(RUN_AS_GRANTED)) {
             try {
-                enqueue(message("run_as_granted", action, user, null, null, message), "run_as_granted");
+                enqueue(message("run_as_granted", action, user, roleNames, null, null, message), "run_as_granted");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [run_as_granted]", e);
             }
@@ -573,10 +568,10 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     }
 
     @Override
-    public void runAsDenied(User user, String action, TransportMessage message) {
+    public void runAsDenied(User user, String action, TransportMessage message, String[] roleNames) {
         if (events.contains(RUN_AS_DENIED)) {
             try {
-                enqueue(message("run_as_denied", action, user, null, null, message), "run_as_denied");
+                enqueue(message("run_as_denied", action, user, roleNames, null, null, message), "run_as_denied");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [run_as_denied]", e);
             }
@@ -584,17 +579,17 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
     }
 
     @Override
-    public void runAsDenied(User user, RestRequest request) {
+    public void runAsDenied(User user, RestRequest request, String[] roleNames) {
         if (events.contains(RUN_AS_DENIED)) {
             try {
-                enqueue(message("run_as_denied", null, user, request), "run_as_denied");
+                enqueue(message("run_as_denied", null, user, roleNames, request), "run_as_denied");
             } catch (Exception e) {
                 logger.warn("failed to index audit event: [run_as_denied]", e);
             }
         }
     }
 
-    private Message message(String type, @Nullable String action, @Nullable User user, @Nullable String realm,
+    private Message message(String type, @Nullable String action, @Nullable User user, @Nullable String[] roleNames, @Nullable String realm,
                             @Nullable Set<String> indices, TransportMessage message) throws Exception {
 
         Message msg = new Message().start();
@@ -617,6 +612,9 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
             } else {
                 msg.builder.field(Field.PRINCIPAL, user.principal());
             }
+            if (roleNames != null) {
+                msg.builder.array(Field.ROLE_NAMES, roleNames);
+            }
         }
         if (indices != null) {
             msg.builder.array(Field.INDICES, indices.toArray(Strings.EMPTY_ARRAY));
@@ -689,7 +687,8 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
         return msg.end();
     }
 
-    private Message message(String type, String realm, User user, RestRequest request) throws Exception {
+    private Message message(String type, @Nullable String realm, @Nullable User user, @Nullable String[] roleNames, RestRequest request)
+            throws Exception {
 
         Message msg = new Message().start();
         common("rest", type, msg.builder);
@@ -701,6 +700,9 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
             } else {
                 msg.builder.field(Field.PRINCIPAL, user.principal());
             }
+            if (roleNames != null) {
+                msg.builder.array(Field.ROLE_NAMES, roleNames);
+            }
         }
         if (realm != null) {
             msg.builder.field(Field.REALM, realm);
@@ -1054,6 +1056,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail {
         String ORIGIN_ADDRESS = "origin_address";
         String ORIGIN_TYPE = "origin_type";
         String PRINCIPAL = "principal";
+        String ROLE_NAMES = "roles";
         String RUN_AS_PRINCIPAL = "run_as_principal";
         String RUN_BY_PRINCIPAL = "run_by_principal";
         String ACTION = "action";

plugin/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -6,6 +6,8 @@
 package org.elasticsearch.xpack.security.audit.logfile;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
@@ -23,7 +25,6 @@ import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.authc.AuthenticationToken;
-import org.elasticsearch.xpack.security.authz.privilege.SystemPrivilege;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 import org.elasticsearch.xpack.security.user.SystemUser;
@@ -37,10 +38,12 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString;
+import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString;
 import static org.elasticsearch.xpack.security.Security.setting;
 import static org.elasticsearch.xpack.security.audit.AuditLevel.ACCESS_DENIED;
 import static org.elasticsearch.xpack.security.audit.AuditLevel.ACCESS_GRANTED;
@@ -58,7 +61,7 @@ import static org.elasticsearch.xpack.security.audit.AuditLevel.parse;
 import static org.elasticsearch.xpack.security.audit.AuditUtil.indices;
 import static org.elasticsearch.xpack.security.audit.AuditUtil.restRequestContent;
 
-public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
+public class LoggingAuditTrail extends AbstractComponent implements AuditTrail, ClusterStateListener {
 
     public static final String NAME = "logfile";
     public static final Setting<Boolean> HOST_ADDRESS_SETTING =
@@ -85,12 +88,10 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
             Setting.boolSetting(setting("audit.logfile.events.emit_request_body"), false, Property.NodeScope);
 
     private final Logger logger;
-    private final ClusterService clusterService;
-    private final ThreadContext threadContext;
     private final EnumSet<AuditLevel> events;
     private final boolean includeRequestBody;
-
-    private String prefix;
+    private final ThreadContext threadContext;
+    volatile LocalNodeInfo localNodeInfo;
 
     @Override
     public String name() {
@@ -104,28 +105,22 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     LoggingAuditTrail(Settings settings, ClusterService clusterService, Logger logger, ThreadContext threadContext) {
         super(settings);
         this.logger = logger;
-        this.clusterService = clusterService;
-        this.threadContext = threadContext;
         this.events = parse(INCLUDE_EVENT_SETTINGS.get(settings), EXCLUDE_EVENT_SETTINGS.get(settings));
         this.includeRequestBody = INCLUDE_REQUEST_BODY.get(settings);
-    }
-
-    private String getPrefix() {
-        if (prefix == null) {
-            prefix = resolvePrefix(settings, clusterService.localNode());
-        }
-        return prefix;
+        this.threadContext = threadContext;
+        this.localNodeInfo = new LocalNodeInfo(settings, null);
+        clusterService.addListener(this);
     }
 
     @Override
     public void authenticationSuccess(String realm, User user, RestRequest request) {
         if (events.contains(AUTHENTICATION_SUCCESS)) {
             if (includeRequestBody) {
-                logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}], request_body=[{}]", getPrefix(),
-                        principal(user), realm, request.uri(), request.params(), restRequestContent(request));
+                logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}], request_body=[{}]",
+                        localNodeInfo.prefix, principal(user), realm, request.uri(), request.params(), restRequestContent(request));
             } else {
-                logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]", getPrefix(), principal(user), realm,
-                        request.uri(), request.params());
+                logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]", localNodeInfo.prefix,
+                        principal(user), realm, request.uri(), request.params());
             }
         }
     }
@@ -133,8 +128,9 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     @Override
     public void authenticationSuccess(String realm, User user, String action, TransportMessage message) {
         if (events.contains(AUTHENTICATION_SUCCESS)) {
-            logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], request=[{}]", getPrefix(),
-                    originAttributes(message, clusterService.localNode(), threadContext), principal(user), realm, action,
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
+            logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], request=[{}]",
+                    localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user), realm, action,
                     message.getClass().getSimpleName());
         }
     }
@@ -143,13 +139,14 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void anonymousAccessDenied(String action, TransportMessage message) {
         if (events.contains(ANONYMOUS_ACCESS_DENIED)) {
             String indices = indicesString(message);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
             if (indices != null) {
-                logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], indices=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), action, indices,
+                logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], indices=[{}], request=[{}]",
+                        localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, indices,
                         message.getClass().getSimpleName());
             } else {
-                logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), action, message.getClass().getSimpleName());
+                logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName());
             }
         }
     }
@@ -158,10 +155,11 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void anonymousAccessDenied(RestRequest request) {
         if (events.contains(ANONYMOUS_ACCESS_DENIED)) {
             if (includeRequestBody) {
-                logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}], request_body=[{}]", getPrefix(),
+                logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}], request_body=[{}]", localNodeInfo.prefix,
                         hostAttributes(request), request.uri(), restRequestContent(request));
             } else {
-                logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]", getPrefix(), hostAttributes(request), request.uri());
+                logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]", localNodeInfo.prefix, hostAttributes(request),
+                        request.uri());
             }
         }
     }
@@ -170,13 +168,14 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationFailed(AuthenticationToken token, String action, TransportMessage message) {
         if (events.contains(AUTHENTICATION_FAILED)) {
             String indices = indicesString(message);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
             if (indices != null) {
                 logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], indices=[{}], request=[{}]",
-                        getPrefix(), originAttributes(message, clusterService.localNode(), threadContext), token.principal(),
-                        action, indices, message.getClass().getSimpleName());
+                        localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, indices,
+                        message.getClass().getSimpleName());
             } else {
-                logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), token.principal(), action,
+                logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], request=[{}]",
+                        localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action,
                         message.getClass().getSimpleName());
             }
 
@@ -187,10 +186,11 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationFailed(RestRequest request) {
         if (events.contains(AUTHENTICATION_FAILED)) {
             if (includeRequestBody) {
-                logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}], request_body=[{}]", getPrefix(), hostAttributes(request),
-                        request.uri(), restRequestContent(request));
+                logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}], request_body=[{}]", localNodeInfo.prefix,
+                        hostAttributes(request), request.uri(), restRequestContent(request));
             } else {
-                logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]", getPrefix(), hostAttributes(request), request.uri());
+                logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]", localNodeInfo.prefix, hostAttributes(request),
+                        request.uri());
             }
         }
     }
@@ -199,13 +199,13 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationFailed(String action, TransportMessage message) {
         if (events.contains(AUTHENTICATION_FAILED)) {
             String indices = indicesString(message);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
             if (indices != null) {
-                logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], indices=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), action, indices,
-                        message.getClass().getSimpleName());
+                logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], indices=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, message, localNodeInfo), action, indices, message.getClass().getSimpleName());
             } else {
-                logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), action, message.getClass().getSimpleName());
+                logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName());
             }
         }
     }
@@ -214,11 +214,11 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationFailed(AuthenticationToken token, RestRequest request) {
         if (events.contains(AUTHENTICATION_FAILED)) {
             if (includeRequestBody) {
-                logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}], request_body=[{}]", getPrefix(),
+                logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}], request_body=[{}]", localNodeInfo.prefix,
                         hostAttributes(request), token.principal(), request.uri(), restRequestContent(request));
             } else {
-                logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]", getPrefix(), hostAttributes(request),
-                        token.principal(), request.uri());
+                logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]", localNodeInfo.prefix,
+                        hostAttributes(request), token.principal(), request.uri());
             }
         }
     }
@@ -227,14 +227,15 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void authenticationFailed(String realm, AuthenticationToken token, String action, TransportMessage message) {
         if (events.contains(REALM_AUTHENTICATION_FAILED)) {
             String indices = indicesString(message);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
             if (indices != null) {
                 logger.info("{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], indices=[{}], " +
-                                "request=[{}]", getPrefix(), realm, originAttributes(message, clusterService.localNode(), threadContext),
+                        "request=[{}]", localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo),
                         token.principal(), action, indices, message.getClass().getSimpleName());
             } else {
                 logger.info("{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], request=[{}]",
-                        getPrefix(), realm, originAttributes(message, clusterService.localNode(), threadContext), token.principal(),
-                        action, message.getClass().getSimpleName());
+                        localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo), token.principal(), action,
+                        message.getClass().getSimpleName());
             }
         }
     }
@@ -244,45 +245,51 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
         if (events.contains(REALM_AUTHENTICATION_FAILED)) {
             if (includeRequestBody) {
                 logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}], request_body=[{}]",
-                        getPrefix(), realm, hostAttributes(request), token.principal(), request.uri(), restRequestContent(request));
+                        localNodeInfo.prefix, realm, hostAttributes(request), token.principal(), request.uri(),
+                        restRequestContent(request));
             } else {
-                logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]", getPrefix(),
+                logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]", localNodeInfo.prefix,
                         realm, hostAttributes(request), token.principal(), request.uri());
             }
         }
     }
 
     @Override
-    public void accessGranted(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices) {
-        final boolean isSystem = (SystemUser.is(user) && SystemPrivilege.INSTANCE.predicate().test(action)) || XPackUser.is(user);
+    public void accessGranted(User user, String action, TransportMessage message, String[] roleNames,
+                              @Nullable Set<String> specificIndices) {
+        final boolean isSystem = SystemUser.is(user) || XPackUser.is(user);
         final boolean logSystemAccessGranted = isSystem && events.contains(SYSTEM_ACCESS_GRANTED);
         final boolean shouldLog = logSystemAccessGranted || (isSystem == false && events.contains(ACCESS_GRANTED));
         if (shouldLog) {
             String indices = specificIndices == null ? indicesString(message) : collectionToCommaDelimitedString(specificIndices);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
             if (indices != null) {
-                logger.info("{}[transport] [access_granted]\t{}, {}, action=[{}], indices=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), principal(user), action, indices,
-                        message.getClass().getSimpleName());
+                logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]",
+                        localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user),
+                        arrayToCommaDelimitedString(roleNames), action, indices, message.getClass().getSimpleName());
             } else {
-                logger.info("{}[transport] [access_granted]\t{}, {}, action=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), principal(user), action,
-                        message.getClass().getSimpleName());
+                logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, message, localNodeInfo), principal(user), arrayToCommaDelimitedString(roleNames),
+                        action, message.getClass().getSimpleName());
             }
         }
     }
 
     @Override
-    public void accessDenied(User user, String action, TransportMessage message, @Nullable Set<String> specificIndices) {
+    public void accessDenied(User user, String action, TransportMessage message, String[] roleNames,
+                             @Nullable Set<String> specificIndices) {
         if (events.contains(ACCESS_DENIED)) {
             String indices = specificIndices == null ? indicesString(message) : collectionToCommaDelimitedString(specificIndices);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
+
             if (indices != null) {
-                logger.info("{}[transport] [access_denied]\t{}, {}, action=[{}], indices=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), principal(user), action, indices,
-                        message.getClass().getSimpleName());
+                logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]",
+                        localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user),
+                        arrayToCommaDelimitedString(roleNames), action, indices, message.getClass().getSimpleName());
             } else {
-                logger.info("{}[transport] [access_denied]\t{}, {}, action=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), principal(user), action,
-                        message.getClass().getSimpleName());
+                logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, message, localNodeInfo), principal(user), arrayToCommaDelimitedString(roleNames),
+                        action, message.getClass().getSimpleName());
             }
         }
     }
@@ -291,10 +298,10 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void tamperedRequest(RestRequest request) {
         if (events.contains(TAMPERED_REQUEST)) {
             if (includeRequestBody) {
-                logger.info("{}[rest] [tampered_request]\t{}, uri=[{}], request_body=[{}]", getPrefix(), hostAttributes(request),
-                        request.uri(), restRequestContent(request));
+                logger.info("{}[rest] [tampered_request]\t{}, uri=[{}], request_body=[{}]", localNodeInfo.prefix,
+                        hostAttributes(request), request.uri(), restRequestContent(request));
             } else {
-                logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]", getPrefix(), hostAttributes(request), request.uri());
+                logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]", localNodeInfo.prefix, hostAttributes(request), request.uri());
             }
         }
     }
@@ -303,14 +310,13 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void tamperedRequest(String action, TransportMessage message) {
         if (events.contains(TAMPERED_REQUEST)) {
             String indices = indicesString(message);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
             if (indices != null) {
-                logger.info("{}[transport] [tampered_request]\t{}, action=[{}], indices=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), action, indices,
-                        message.getClass().getSimpleName());
+                logger.info("{}[transport] [tampered_request]\t{}, action=[{}], indices=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, message, localNodeInfo), action, indices, message.getClass().getSimpleName());
             } else {
-                logger.info("{}[transport] [tampered_request]\t{}, action=[{}], request=[{}]", getPrefix(),
-                        originAttributes(message, clusterService.localNode(), threadContext), action,
-                        message.getClass().getSimpleName());
+                logger.info("{}[transport] [tampered_request]\t{}, action=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName());
             }
         }
     }
@@ -319,13 +325,14 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     public void tamperedRequest(User user, String action, TransportMessage request) {
         if (events.contains(TAMPERED_REQUEST)) {
             String indices = indicesString(request);
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
             if (indices != null) {
-                logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], indices=[{}], request=[{}]", getPrefix(),
-                        originAttributes(request, clusterService.localNode(), threadContext), principal(user), action, indices,
+                logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], indices=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, request, localNodeInfo), principal(user), action, indices,
                         request.getClass().getSimpleName());
             } else {
-                logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], request=[{}]", getPrefix(),
-                        originAttributes(request, clusterService.localNode(), threadContext), principal(user), action,
+                logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], request=[{}]", localNodeInfo.prefix,
+                        originAttributes(threadContext, request, localNodeInfo), principal(user), action,
                         request.getClass().getSimpleName());
             }
         }
@@ -334,46 +341,49 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
     @Override
     public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {
         if (events.contains(CONNECTION_GRANTED)) {
-            logger.info("{}[ip_filter] [connection_granted]\torigin_address=[{}], transport_profile=[{}], rule=[{}]", getPrefix(),
-                    NetworkAddress.format(inetAddress), profile, rule);
+            logger.info("{}[ip_filter] [connection_granted]\torigin_address=[{}], transport_profile=[{}], rule=[{}]",
+                    localNodeInfo.prefix, NetworkAddress.format(inetAddress), profile, rule);
         }
     }
 
     @Override
     public void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {
         if (events.contains(CONNECTION_DENIED)) {
-            logger.info("{}[ip_filter] [connection_denied]\torigin_address=[{}], transport_profile=[{}], rule=[{}]", getPrefix(),
-                    NetworkAddress.format(inetAddress), profile, rule);
+            logger.info("{}[ip_filter] [connection_denied]\torigin_address=[{}], transport_profile=[{}], rule=[{}]",
+                    localNodeInfo.prefix, NetworkAddress.format(inetAddress), profile, rule);
         }
     }
 
     @Override
-    public void runAsGranted(User user, String action, TransportMessage message) {
+    public void runAsGranted(User user, String action, TransportMessage message, String[] roleNames) {
         if (events.contains(RUN_AS_GRANTED)) {
-            logger.info("{}[transport] [run_as_granted]\t{}, principal=[{}], run_as_principal=[{}], action=[{}], request=[{}]",
-                getPrefix(), originAttributes(message, clusterService.localNode(), threadContext), user.authenticatedUser().principal(),
-                    user.principal(), action, message.getClass().getSimpleName());
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
+            logger.info("{}[transport] [run_as_granted]\t{}, principal=[{}], run_as_principal=[{}], roles=[{}], action=[{}], request=[{}]",
+                    localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), user.authenticatedUser().principal(),
+                    user.principal(), arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName());
         }
     }
 
     @Override
-    public void runAsDenied(User user, String action, TransportMessage message) {
+    public void runAsDenied(User user, String action, TransportMessage message, String[] roleNames) {
         if (events.contains(RUN_AS_DENIED)) {
-            logger.info("{}[transport] [run_as_denied]\t{}, principal=[{}], run_as_principal=[{}], action=[{}], request=[{}]",
-                getPrefix(), originAttributes(message, clusterService.localNode(), threadContext), user.authenticatedUser().principal(),
-                    user.principal(), action, message.getClass().getSimpleName());
+            final LocalNodeInfo localNodeInfo = this.localNodeInfo;
+            logger.info("{}[transport] [run_as_denied]\t{}, principal=[{}], run_as_principal=[{}], roles=[{}], action=[{}], request=[{}]",
+                    localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), user.authenticatedUser().principal(),
+                    user.principal(), arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName());
         }
     }
 
     @Override
-    public void runAsDenied(User user, RestRequest request) {
+    public void runAsDenied(User user, RestRequest request, String[] roleNames) {
         if (events.contains(RUN_AS_DENIED)) {
             if (includeRequestBody) {
-                logger.info("{}[rest] [run_as_denied]\t{}, principal=[{}], uri=[{}], request_body=[{}]", getPrefix(),
-                    hostAttributes(request), user.principal(), request.uri(), restRequestContent(request));
+                logger.info("{}[rest] [run_as_denied]\t{}, principal=[{}], roles=[{}], uri=[{}], request_body=[{}]", localNodeInfo.prefix,
+                        hostAttributes(request), user.principal(), arrayToCommaDelimitedString(roleNames), request.uri(),
+                        restRequestContent(request));
             } else {
-                logger.info("{}[rest] [run_as_denied]\t{}, principal=[{}], uri=[{}]", getPrefix(),
-                        hostAttributes(request), user.principal(), request.uri());
+                logger.info("{}[rest] [run_as_denied]\t{}, principal=[{}], roles=[{}], uri=[{}]", localNodeInfo.prefix,
+                        hostAttributes(request), user.principal(), arrayToCommaDelimitedString(roleNames), request.uri());
             }
         }
     }
@@ -389,56 +399,29 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
         return "origin_address=[" + formattedAddress + "]";
     }
 
-    static String originAttributes(TransportMessage message, DiscoveryNode localNode, ThreadContext threadContext) {
-        StringBuilder builder = new StringBuilder();
-
-        // first checking if the message originated in a rest call
-        InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext);
-        if (restAddress != null) {
-            builder.append("origin_type=[rest], origin_address=[").
-                    append(NetworkAddress.format(restAddress.getAddress())).
-                    append("]");
-            return builder.toString();
-        }
-
-        // we'll see if was originated in a remote node
-        TransportAddress address = message.remoteAddress();
-        if (address != null) {
-            builder.append("origin_type=[transport], ");
-            builder.append("origin_address=[").
-                        append(NetworkAddress.format(address.address().getAddress())).
-                        append("]");
-            return builder.toString();
-        }
-
-        // the call was originated locally on this node
-        return builder.append("origin_type=[local_node], origin_address=[")
-                .append(localNode.getHostAddress())
-                .append("]")
-                .toString();
+    protected static String originAttributes(ThreadContext threadContext, TransportMessage message, LocalNodeInfo localNodeInfo) {
+        return restOriginTag(threadContext).orElse(transportOriginTag(message).orElse(localNodeInfo.localOriginTag));
     }
 
-    static String resolvePrefix(Settings settings, DiscoveryNode localNode) {
-        StringBuilder builder = new StringBuilder();
-        if (HOST_ADDRESS_SETTING.get(settings)) {
-            String address = localNode.getHostAddress();
-            if (address != null) {
-                builder.append("[").append(address).append("] ");
-            }
+    private static Optional<String> restOriginTag(ThreadContext threadContext) {
+        InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext);
+        if (restAddress == null) {
+            return Optional.empty();
         }
-        if (HOST_NAME_SETTING.get(settings)) {
-            String hostName = localNode.getHostName();
-            if (hostName != null) {
-                builder.append("[").append(hostName).append("] ");
-            }
+        return Optional.of(new StringBuilder("origin_type=[rest], origin_address=[").append(NetworkAddress.format(restAddress.getAddress()))
+                .append("]")
+                .toString());
+    }
+
+    private static Optional<String> transportOriginTag(TransportMessage message) {
+        TransportAddress address = message.remoteAddress();
+        if (address == null) {
+            return Optional.empty();
         }
-        if (NODE_NAME_SETTING.get(settings)) {
-            String name = settings.get("name");
-            if (name != null) {
-                builder.append("[").append(name).append("] ");
-            }
-        }
-        return builder.toString();
+        return Optional.of(
+                new StringBuilder("origin_type=[transport], origin_address=[").append(NetworkAddress.format(address.address().getAddress()))
+                        .append("]")
+                        .toString());
     }
 
     static String indicesString(TransportMessage message) {
@@ -463,4 +446,62 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail {
         settings.add(EXCLUDE_EVENT_SETTINGS);
         settings.add(INCLUDE_REQUEST_BODY);
     }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        updateLocalNodeInfo(event.state().getNodes().getLocalNode());
+    }
+
+    void updateLocalNodeInfo(DiscoveryNode newLocalNode) {
+        // check if local node changed
+        final DiscoveryNode localNode = localNodeInfo.localNode;
+        if (localNode == null || localNode.equals(newLocalNode) == false) {
+            // no need to synchronize, called only from the cluster state applier thread
+            localNodeInfo = new LocalNodeInfo(settings, newLocalNode);
+        }
+    }
+
+    protected static class LocalNodeInfo {
+        private final DiscoveryNode localNode;
+        private final String prefix;
+        private final String localOriginTag;
+
+        LocalNodeInfo(Settings settings, @Nullable DiscoveryNode newLocalNode) {
+            this.localNode = newLocalNode;
+            this.prefix = resolvePrefix(settings, newLocalNode);
+            this.localOriginTag = localOriginTag(newLocalNode);
+        }
+
+        static String resolvePrefix(Settings settings, @Nullable DiscoveryNode localNode) {
+            final StringBuilder builder = new StringBuilder();
+            if (HOST_ADDRESS_SETTING.get(settings)) {
+                String address = localNode != null ? localNode.getHostAddress() : null;
+                if (address != null) {
+                    builder.append("[").append(address).append("] ");
+                }
+            }
+            if (HOST_NAME_SETTING.get(settings)) {
+                String hostName = localNode != null ? localNode.getHostName() : null;
+                if (hostName != null) {
+                    builder.append("[").append(hostName).append("] ");
+                }
+            }
+            if (NODE_NAME_SETTING.get(settings)) {
+                String name = settings.get("name");
+                if (name != null) {
+                    builder.append("[").append(name).append("] ");
+                }
+            }
+            return builder.toString();
+        }
+
+        private static String localOriginTag(@Nullable DiscoveryNode localNode) {
+            if (localNode == null) {
+                return "origin_type=[local_node]";
+            }
+            return new StringBuilder("origin_type=[local_node], origin_address=[").append(localNode.getHostAddress())
+                    .append("]")
+                    .toString();
+        }
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -34,6 +34,7 @@ import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.security.authz.permission.Role;
 import org.elasticsearch.xpack.security.support.Exceptions;
 import org.elasticsearch.xpack.security.user.AnonymousUser;
 import org.elasticsearch.xpack.security.user.User;
@@ -531,7 +532,7 @@ public class AuthenticationService extends AbstractComponent {
 
         @Override
         ElasticsearchSecurityException runAsDenied(User user, AuthenticationToken token) {
-            auditTrail.runAsDenied(user, action, message);
+            auditTrail.runAsDenied(user, action, message, Role.EMPTY.names());
             return failureHandler.failedAuthentication(message, token, action, threadContext);
         }
 
@@ -593,7 +594,7 @@ public class AuthenticationService extends AbstractComponent {
 
         @Override
         ElasticsearchSecurityException runAsDenied(User user, AuthenticationToken token) {
-            auditTrail.runAsDenied(user, request);
+            auditTrail.runAsDenied(user, request, Role.EMPTY.names());
             return failureHandler.failedAuthentication(request, token, threadContext);
         }
 

plugin/src/main/java/org/elasticsearch/xpack/security/authc/RealmSettings.java

@@ -16,6 +16,7 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import org.elasticsearch.common.settings.AbstractScopedSettings;
+import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.extensions.XPackExtension;
@@ -167,6 +168,11 @@ public class RealmSettings {
             // perfectly aligned
             return;
         }
+
+        // Don't validate secure settings because they might have been cleared already
+        settings = Settings.builder().put(settings, false).build();
+        validSettings.removeIf(s -> s instanceof SecureSetting);
+
         Set<Setting<?>> settingSet = new HashSet<>(validSettings);
         settingSet.add(TYPE_SETTING);
         settingSet.add(ENABLED_SETTING);

plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java

@@ -88,6 +88,7 @@ public class AuthorizationService extends AbstractComponent {
     public static final String INDICES_PERMISSIONS_KEY = "_indices_permissions";
     public static final String INDICES_PERMISSIONS_RESOLVER_KEY = "_indices_permissions_resolver";
     public static final String ORIGINATING_ACTION_KEY = "_originating_action_name";
+    public static final String ROLE_NAMES_KEY = "_effective_role_names";
 
     private static final Predicate<String> MONITOR_INDEX_PREDICATE = IndexPrivilege.MONITOR.predicate();
     private static final Predicate<String> SAME_USER_PRIVILEGE = Automatons.predicate(
@@ -153,12 +154,13 @@ public class AuthorizationService extends AbstractComponent {
 
         // first we need to check if the user is the system. If it is, we'll just authorize the system access
         if (SystemUser.is(authentication.getUser())) {
-            if (SystemUser.isAuthorized(action) && SystemUser.is(authentication.getUser())) {
-                setIndicesAccessControl(IndicesAccessControl.ALLOW_ALL);
-                grant(authentication, action, request, null);
+            if (SystemUser.isAuthorized(action)) {
+                putTransientIfNonExisting(INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
+                putTransientIfNonExisting(ROLE_NAMES_KEY, new String[] { SystemUser.ROLE_NAME });
+                grant(authentication, action, request, new String[] { SystemUser.ROLE_NAME }, null);
                 return;
             }
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, new String[] { SystemUser.ROLE_NAME }, null);
         }
 
         // get the roles of the authenticated user, which may be different than the effective
@@ -170,29 +172,30 @@ public class AuthorizationService extends AbstractComponent {
             // if we are running as a user we looked up then the authentication must contain a lookedUpBy. If it doesn't then this user
             // doesn't really exist but the authc service allowed it through to avoid leaking users that exist in the system
             if (authentication.getLookedUpBy() == null) {
-                throw denyRunAs(authentication, action, request);
+                throw denyRunAs(authentication, action, request, permission.names());
             } else if (permission.runAs().check(authentication.getUser().principal())) {
-                grantRunAs(authentication, action, request);
+                grantRunAs(authentication, action, request, permission.names());
                 permission = runAsRole;
             } else {
-                throw denyRunAs(authentication, action, request);
+                throw denyRunAs(authentication, action, request, permission.names());
             }
         }
+        putTransientIfNonExisting(ROLE_NAMES_KEY, permission.names());
 
         // first, we'll check if the action is a cluster action. If it is, we'll only check it against the cluster permissions
         if (ClusterPrivilege.ACTION_MATCHER.test(action)) {
             ClusterPermission cluster = permission.cluster();
             if (cluster.check(action) || checkSameUserPermissions(action, request, authentication)) {
-                setIndicesAccessControl(IndicesAccessControl.ALLOW_ALL);
-                grant(authentication, action, request, null);
+                putTransientIfNonExisting(INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
+                grant(authentication, action, request, permission.names(), null);
                 return;
             }
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, permission.names(), null);
         }
 
         // ok... this is not a cluster action, let's verify it's an indices action
         if (!IndexPrivilege.ACTION_MATCHER.test(action)) {
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, permission.names(), null);
         }
 
         //composite actions are explicitly listed and will be authorized at the sub-request / shard level
@@ -203,16 +206,16 @@ public class AuthorizationService extends AbstractComponent {
             }
             // we check if the user can execute the action, without looking at indices, which will be authorized at the shard level
             if (permission.indices().check(action)) {
-                grant(authentication, action, request, null);
+                grant(authentication, action, request, permission.names(), null);
                 return;
             }
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, permission.names(), null);
         } else if (isDelayedIndicesAction(action)) {
             /* We check now if the user can execute the action without looking at indices.
              * The action is itself responsible for checking if the user can access the
              * indices when it runs. */
             if (permission.indices().check(action)) {
-                grant(authentication, action, request, null);
+                grant(authentication, action, request, permission.names(), null);
 
                 /* Now that we know the user can run the action we need to build a function
                  * that we can use later to fetch the user's actual permissions for an
@@ -244,19 +247,33 @@ public class AuthorizationService extends AbstractComponent {
                     try {
                         resolvedIndices = indicesAndAliasesResolver.resolve(proxy, metaData, authorizedIndices);
                     } catch (Exception e) {
-                        denial(authentication, action, finalRequest, specificIndices);
+                        denial(authentication, action, finalRequest, finalPermission.names(), specificIndices);
                         throw e;
                     }
 
                     Set<String> localIndices = new HashSet<>(resolvedIndices.getLocal());
-                    IndicesAccessControl indicesAccessControl = authorizeIndices(action, finalRequest, localIndices, specificIndices,
-                            authentication, finalPermission, metaData);
-                    grant(authentication, action, finalRequest, specificIndices);
+
+                    IndicesAccessControl indicesAccessControl = finalPermission.authorize(action, localIndices,
+                            metaData, fieldPermissionsCache);
+                    if (!indicesAccessControl.isGranted()) {
+                        throw denial(authentication, action, finalRequest, finalPermission.names(), specificIndices);
+                    }
+                    if (hasSecurityIndexAccess(indicesAccessControl)
+                            && MONITOR_INDEX_PREDICATE.test(action) == false
+                            && isSuperuser(authentication.getUser()) == false) {
+                        // only the superusers are allowed to work with this index, but we should allow indices monitoring actions
+                        // through for debuggingpurposes. These monitor requests also sometimes resolve indices concretely and
+                        // then requests them
+                        logger.debug("user [{}] attempted to directly perform [{}] against the security index [{}]",
+                                authentication.getUser().principal(), action, SecurityLifecycleService.SECURITY_INDEX_NAME);
+                        throw denial(authentication, action, finalRequest, finalPermission.names(), specificIndices);
+                    }
+                    grant(authentication, action, finalRequest, finalPermission.names(), specificIndices);
                     return indicesAccessControl;
                 });
                 return;
             }
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, permission.names(), null);
         } else if (isTranslatedToBulkAction(action)) {
             if (request instanceof CompositeIndicesRequest == false) {
                 throw new IllegalStateException("Bulk translated actions must implement " + CompositeIndicesRequest.class.getSimpleName()
@@ -264,10 +281,10 @@ public class AuthorizationService extends AbstractComponent {
             }
             // we check if the user can execute the action, without looking at indices, which will be authorized at the shard level
             if (permission.indices().check(action)) {
-                grant(authentication, action, request, null);
+                grant(authentication, action, request, permission.names(), null);
                 return;
             }
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, permission.names(), null);
         } else if (TransportActionProxy.isProxyAction(action)) {
             // we authorize proxied actions once they are "unwrapped" on the next node
             if (TransportActionProxy.isProxyRequest(originalRequest) == false) {
@@ -275,12 +292,12 @@ public class AuthorizationService extends AbstractComponent {
                         + action + "] is a proxy action");
             }
             if (permission.indices().check(action)) {
-                grant(authentication, action, request, null);
+                grant(authentication, action, request, permission.names(), null);
                 return;
             } else {
                 // we do this here in addition to the denial below since we might run into an assertion on scroll request below if we
                 // don't have permission to read cross cluster but wrap a scroll request.
-                throw denial(authentication, action, request, null);
+                throw denial(authentication, action, request, permission.names(), null);
             }
         }
 
@@ -299,18 +316,18 @@ public class AuthorizationService extends AbstractComponent {
                 // index and if they cannot, we can fail the request early before we allow the execution of the action and in
                 // turn the shard actions
                 if (SearchScrollAction.NAME.equals(action) && permission.indices().check(action) == false) {
-                    throw denial(authentication, action, request, null);
+                    throw denial(authentication, action, request, permission.names(), null);
                 } else {
                     // we store the request as a transient in the ThreadContext in case of a authorization failure at the shard
                     // level. If authorization fails we will audit a access_denied message and will use the request to retrieve
                     // information such as the index and the incoming address of the request
-                    grant(authentication, action, request, null);
+                    grant(authentication, action, request, permission.names(), null);
                     return;
                 }
             } else {
                 assert false :
                         "only scroll related requests are known indices api that don't support retrieving the indices they relate to";
-                throw denial(authentication, action, request, null);
+                throw denial(authentication, action, request, permission.names(), null);
             }
         }
 
@@ -320,33 +337,45 @@ public class AuthorizationService extends AbstractComponent {
         // If this request does not allow remote indices
         // then the user must have permission to perform this action on at least 1 local index
         if (allowsRemoteIndices == false && permission.indices().check(action) == false) {
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, permission.names(), null);
         }
 
         final MetaData metaData = clusterService.state().metaData();
         final AuthorizedIndices authorizedIndices = new AuthorizedIndices(authentication.getUser(), permission, action, metaData);
-        final ResolvedIndices resolvedIndices = resolveIndexNames(authentication, action, request, request, metaData, authorizedIndices);
+        final ResolvedIndices resolvedIndices = resolveIndexNames(authentication, action, request, request,
+                metaData, authorizedIndices, permission);
         assert !resolvedIndices.isEmpty()
                 : "every indices request needs to have its indices set thus the resolved indices must not be empty";
 
         // If this request does reference any remote indices
         // then the user must have permission to perform this action on at least 1 local index
         if (resolvedIndices.getRemote().isEmpty() && permission.indices().check(action) == false) {
-            throw denial(authentication, action, request, null);
+            throw denial(authentication, action, request, permission.names(), null);
         }
 
         //all wildcard expressions have been resolved and only the security plugin could have set '-*' here.
         //'-*' matches no indices so we allow the request to go through, which will yield an empty response
         if (resolvedIndices.isNoIndicesPlaceholder()) {
-            setIndicesAccessControl(IndicesAccessControl.ALLOW_NO_INDICES);
-            grant(authentication, action, request, null);
+            putTransientIfNonExisting(INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_NO_INDICES);
+            grant(authentication, action, request, permission.names(), null);
             return;
         }
 
         final Set<String> localIndices = new HashSet<>(resolvedIndices.getLocal());
-        IndicesAccessControl indicesAccessControl = authorizeIndices(action, request, localIndices, null, authentication,
-                permission, metaData);
-        setIndicesAccessControl(indicesAccessControl);
+        IndicesAccessControl indicesAccessControl = permission.authorize(action, localIndices, metaData, fieldPermissionsCache);
+        if (!indicesAccessControl.isGranted()) {
+            throw denial(authentication, action, request, permission.names(), null);
+        } else if (hasSecurityIndexAccess(indicesAccessControl)
+                && MONITOR_INDEX_PREDICATE.test(action) == false
+                && isSuperuser(authentication.getUser()) == false) {
+            // only the XPackUser is allowed to work with this index, but we should allow indices monitoring actions through for debugging
+            // purposes. These monitor requests also sometimes resolve indices concretely and then requests them
+            logger.debug("user [{}] attempted to directly perform [{}] against the security index [{}]",
+                    authentication.getUser().principal(), action, SecurityLifecycleService.SECURITY_INDEX_NAME);
+            throw denial(authentication, action, request, permission.names(), null);
+        } else {
+            putTransientIfNonExisting(INDICES_PERMISSIONS_KEY, indicesAccessControl);
+        }
 
         //if we are creating an index we need to authorize potential aliases created at the same time
         if (IndexPrivilege.CREATE_INDEX_MATCHER.test(action)) {
@@ -359,7 +388,7 @@ public class AuthorizationService extends AbstractComponent {
                 }
                 indicesAccessControl = permission.authorize("indices:admin/aliases", aliasesAndIndices, metaData, fieldPermissionsCache);
                 if (!indicesAccessControl.isGranted()) {
-                    throw denial(authentication, "indices:admin/aliases", request, null);
+                    throw denial(authentication, "indices:admin/aliases", request, permission.names(), null);
                 }
                 // no need to re-add the indicesAccessControl in the context,
                 // because the create index call doesn't do anything FLS or DLS
@@ -374,7 +403,7 @@ public class AuthorizationService extends AbstractComponent {
             authorizeBulkItems(authentication, (BulkShardRequest) request, permission, metaData, localIndices, authorizedIndices);
         }
 
-        grant(authentication, action, originalRequest, null);
+        grant(authentication, action, originalRequest, permission.names(), null);
     }
 
     private boolean hasSecurityIndexAccess(IndicesAccessControl indicesAccessControl) {
@@ -389,13 +418,17 @@ public class AuthorizationService extends AbstractComponent {
 
     /**
      * Performs authorization checks on the items within a {@link BulkShardRequest}.
-     * This inspects the {@link BulkItemRequest items} within the request, computes an <em>implied</em> action for each item's
-     * {@link DocWriteRequest#opType()}, and then checks whether that action is allowed on the targeted index.
-     * Items that fail this checks are {@link BulkItemRequest#abort(String, Exception) aborted}, with an
-     * {@link #denial(Authentication, String, TransportRequest, Set) access denied} exception.
-     * Because a shard level request is for exactly 1 index, and there are a small number of possible item
-     * {@link DocWriteRequest.OpType types}, the number of distinct authorization checks that need to be performed is very small, but the
-     * results must be cached, to avoid adding a high overhead to each bulk request.
+     * This inspects the {@link BulkItemRequest items} within the request, computes
+     * an <em>implied</em> action for each item's {@link DocWriteRequest#opType()},
+     * and then checks whether that action is allowed on the targeted index. Items
+     * that fail this checks are {@link BulkItemRequest#abort(String, Exception)
+     * aborted}, with an
+     * {@link #denial(Authentication, String, TransportRequest, String[], Set) access
+     * denied} exception. Because a shard level request is for exactly 1 index, and
+     * there are a small number of possible item {@link DocWriteRequest.OpType
+     * types}, the number of distinct authorization checks that need to be performed
+     * is very small, but the results must be cached, to avoid adding a high
+     * overhead to each bulk request.
      */
     private void authorizeBulkItems(Authentication authentication, BulkShardRequest request, Role permission,
                                     MetaData metaData, Set<String> indices, AuthorizedIndices authorizedIndices) {
@@ -429,7 +462,7 @@ public class AuthorizationService extends AbstractComponent {
                 return itemAccessControl.isGranted();
             });
             if (granted == false) {
-                item.abort(resolvedIndex, denial(authentication, itemAction, request, null));
+                item.abort(resolvedIndex, denial(authentication, itemAction, request, permission.names(), null));
             }
         }
     }
@@ -454,20 +487,16 @@ public class AuthorizationService extends AbstractComponent {
     }
 
     private ResolvedIndices resolveIndexNames(Authentication authentication, String action, Object indicesRequest,
-                                              TransportRequest mainRequest, MetaData metaData,
-                                              AuthorizedIndices authorizedIndices) {
+                                              TransportRequest mainRequest,MetaData metaData,
+                                              AuthorizedIndices authorizedIndices, Role permission) {
         try {
             return indicesAndAliasesResolver.resolve(indicesRequest, metaData, authorizedIndices);
         } catch (Exception e) {
-            auditTrail.accessDenied(authentication.getUser(), action, mainRequest, null);
+            auditTrail.accessDenied(authentication.getUser(), action, mainRequest, permission.names(), null);
             throw e;
         }
     }
 
-    private void setIndicesAccessControl(IndicesAccessControl accessControl) {
-        putTransientIfNonExisting(INDICES_PERMISSIONS_KEY, accessControl);
-    }
-
     /**
      * Sets a function to resolve {@link IndicesAccessControl} to be used by
      * {@link #isDelayedIndicesAction(String) actions} that do not know their
@@ -514,7 +543,7 @@ public class AuthorizationService extends AbstractComponent {
 
         if (roleNames.isEmpty()) {
             roleActionListener.onResponse(Role.EMPTY);
-        } else if (roleNames.contains(ReservedRolesStore.SUPERUSER_ROLE.name())) {
+        } else if (roleNames.contains(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName())) {
             roleActionListener.onResponse(ReservedRolesStore.SUPERUSER_ROLE);
         } else {
             rolesStore.roles(roleNames, fieldPermissionsCache, roleActionListener);
@@ -562,28 +591,6 @@ public class AuthorizationService extends AbstractComponent {
                 action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
     }
 
-    /**
-     * Authorize some indices, throwing an exception if they are not authorized and returning
-     * the {@link IndicesAccessControl} if they are.
-     */
-    private IndicesAccessControl authorizeIndices(String action, TransportRequest request, Set<String> localIndices,
-            Set<String> specificIndices, Authentication authentication, Role permission, MetaData metaData) {
-        IndicesAccessControl indicesAccessControl = permission.authorize(action, localIndices, metaData, fieldPermissionsCache);
-        if (!indicesAccessControl.isGranted()) {
-            throw denial(authentication, action, request, specificIndices);
-        }
-        if (hasSecurityIndexAccess(indicesAccessControl)
-                && MONITOR_INDEX_PREDICATE.test(action) == false
-                && isSuperuser(authentication.getUser()) == false) {
-            // only the superusers are allowed to work with this index, but we should allow indices monitoring actions through for debugging
-            // purposes. These monitor requests also sometimes resolve indices concretely and then requests them
-            logger.debug("user [{}] attempted to directly perform [{}] against the security index [{}]",
-                    authentication.getUser().principal(), action, SecurityLifecycleService.SECURITY_INDEX_NAME);
-            throw denial(authentication, action, request, specificIndices);
-        }
-        return indicesAccessControl;
-    }
-
     static boolean checkSameUserPermissions(String action, TransportRequest request, Authentication authentication) {
         final boolean actionAllowed = SAME_USER_PRIVILEGE.test(action);
         if (actionAllowed) {
@@ -629,22 +636,24 @@ public class AuthorizationService extends AbstractComponent {
     }
 
     ElasticsearchSecurityException denial(Authentication authentication, String action, TransportRequest request,
-            @Nullable Set<String> specificIndices) {
-        auditTrail.accessDenied(authentication.getUser(), action, request, specificIndices);
+                String[] roleNames, @Nullable Set<String> specificIndices) {
+        auditTrail.accessDenied(authentication.getUser(), action, request, roleNames, specificIndices);
         return denialException(authentication, action);
     }
 
-    private ElasticsearchSecurityException denyRunAs(Authentication authentication, String action, TransportRequest request) {
-        auditTrail.runAsDenied(authentication.getUser(), action, request);
+    private ElasticsearchSecurityException denyRunAs(Authentication authentication, String action, TransportRequest request,
+                                                     String[] roleNames) {
+        auditTrail.runAsDenied(authentication.getUser(), action, request, roleNames);
         return denialException(authentication, action);
     }
 
-    private void grant(Authentication authentication, String action, TransportRequest request, @Nullable Set<String> specificIndices) {
-        auditTrail.accessGranted(authentication.getUser(), action, request, specificIndices);
+    private void grant(Authentication authentication, String action, TransportRequest request,
+                String[] roleNames, @Nullable Set<String> specificIndices) {
+        auditTrail.accessGranted(authentication.getUser(), action, request, roleNames, specificIndices);
     }
 
-    private void grantRunAs(Authentication authentication, String action, TransportRequest request) {
-        auditTrail.runAsGranted(authentication.getUser(), action, request);
+    private void grantRunAs(Authentication authentication, String action, TransportRequest request, String[] roleNames) {
+        auditTrail.runAsGranted(authentication.getUser(), action, request, roleNames);
     }
 
     private ElasticsearchSecurityException denialException(Authentication authentication, String action) {
@@ -665,7 +674,7 @@ public class AuthorizationService extends AbstractComponent {
 
     static boolean isSuperuser(User user) {
         return Arrays.stream(user.roles())
-                .anyMatch(ReservedRolesStore.SUPERUSER_ROLE.name()::equals);
+                .anyMatch(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName()::equals);
     }
 
     public static void addSettings(List<Setting<?>> settings) {

plugin/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java

@@ -16,6 +16,7 @@ import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.Authentication;
 
 import static org.elasticsearch.xpack.security.authz.AuthorizationService.ORIGINATING_ACTION_KEY;
+import static org.elasticsearch.xpack.security.authz.AuthorizationService.ROLE_NAMES_KEY;
 
 /**
  * A {@link SearchOperationListener} that is used to provide authorization for scroll requests.
@@ -59,7 +60,8 @@ public final class SecuritySearchOperationListener implements SearchOperationLis
                 final Authentication originalAuth = searchContext.scrollContext().getFromContext(Authentication.AUTHENTICATION_KEY);
                 final Authentication current = Authentication.getAuthentication(threadContext);
                 final String action = threadContext.getTransient(ORIGINATING_ACTION_KEY);
-                ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, searchContext.id(), action, request);
+                ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, searchContext.id(), action, request,
+                        threadContext.getTransient(ROLE_NAMES_KEY));
             }
         }
     }
@@ -71,7 +73,7 @@ public final class SecuritySearchOperationListener implements SearchOperationLis
      * (or lookup) realm. To work around this we compare the username and the originating realm type.
      */
     static void ensureAuthenticatedUserIsSame(Authentication original, Authentication current, AuditTrailService auditTrailService,
-                                              long id, String action, TransportRequest request) {
+                                              long id, String action, TransportRequest request, String[] roleNames) {
         // this is really a best effort attempt since we cannot guarantee principal uniqueness
         // and realm names can change between nodes.
         final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal());
@@ -90,7 +92,7 @@ public final class SecuritySearchOperationListener implements SearchOperationLis
 
         final boolean sameUser = samePrincipal && sameRealmType;
         if (sameUser == false) {
-            auditTrailService.accessDenied(current.getUser(), action, request, null);
+            auditTrailService.accessDenied(current.getUser(), action, request, roleNames, null);
             throw new SearchContextMissingException(id);
         }
     }

plugin/src/main/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissions.java

@@ -115,7 +115,7 @@ public final class FieldPermissions implements Accountable {
         return a.getNumStates() * 5; // wild guess, better than 0
     }
 
-    static Automaton initializePermittedFieldsAutomaton(FieldPermissionsDefinition fieldPermissionsDefinition) {
+    public static Automaton initializePermittedFieldsAutomaton(FieldPermissionsDefinition fieldPermissionsDefinition) {
         Set<FieldGrantExcludeGroup> groups = fieldPermissionsDefinition.getFieldGrantExcludeGroups();
         assert groups.size() > 0 : "there must always be a single group for field inclusion/exclusion";
         List<Automaton> automatonList =

plugin/src/main/java/org/elasticsearch/xpack/security/authz/permission/Role.java

@@ -27,20 +27,20 @@ public final class Role {
 
     public static final Role EMPTY = Role.builder("__empty").build();
 
-    private final String name;
+    private final String[] names;
     private final ClusterPermission cluster;
     private final IndicesPermission indices;
     private final RunAsPermission runAs;
 
-    Role(String name, ClusterPermission cluster, IndicesPermission indices, RunAsPermission runAs) {
-        this.name = name;
+    Role(String[] names, ClusterPermission cluster, IndicesPermission indices, RunAsPermission runAs) {
+        this.names = names;
         this.cluster = Objects.requireNonNull(cluster);
         this.indices = Objects.requireNonNull(indices);
         this.runAs = Objects.requireNonNull(runAs);
     }
 
-    public String name() {
-        return name;
+    public String[] names() {
+        return names;
     }
 
     public ClusterPermission cluster() {
@@ -55,12 +55,12 @@ public final class Role {
         return runAs;
     }
 
-    public static Builder builder(String name) {
-        return new Builder(name, null);
+    public static Builder builder(String... names) {
+        return new Builder(names, null);
     }
 
-    public static Builder builder(String name, FieldPermissionsCache fieldPermissionsCache) {
-        return new Builder(name, fieldPermissionsCache);
+    public static Builder builder(String[] names, FieldPermissionsCache fieldPermissionsCache) {
+        return new Builder(names, fieldPermissionsCache);
     }
 
     public static Builder builder(RoleDescriptor rd, FieldPermissionsCache fieldPermissionsCache) {
@@ -91,19 +91,19 @@ public final class Role {
 
     public static class Builder {
 
-        private final String name;
+        private final String[] names;
         private ClusterPermission cluster = ClusterPermission.NONE;
         private RunAsPermission runAs = RunAsPermission.NONE;
         private List<IndicesPermission.Group> groups = new ArrayList<>();
         private FieldPermissionsCache fieldPermissionsCache = null;
 
-        private Builder(String name, FieldPermissionsCache fieldPermissionsCache) {
-            this.name = name;
+        private Builder(String[] names, FieldPermissionsCache fieldPermissionsCache) {
+            this.names = names;
             this.fieldPermissionsCache = fieldPermissionsCache;
         }
 
         private Builder(RoleDescriptor rd, @Nullable FieldPermissionsCache fieldPermissionsCache) {
-            this.name = rd.getName();
+            this.names = new String[] { rd.getName() };
             this.fieldPermissionsCache = fieldPermissionsCache;
             if (rd.getClusterPrivileges().length == 0) {
                 cluster = ClusterPermission.NONE;
@@ -140,7 +140,7 @@ public final class Role {
         public Role build() {
             IndicesPermission indices = groups.isEmpty() ? IndicesPermission.NONE :
                     new IndicesPermission(groups.toArray(new IndicesPermission.Group[groups.size()]));
-            return new Role(name, cluster, indices, runAs);
+            return new Role(names, cluster, indices, runAs);
         }
 
         static List<IndicesPermission.Group> convertFromIndicesPrivileges(RoleDescriptor.IndicesPrivileges[] indicesPrivileges,

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -5,35 +5,9 @@
  */
 package org.elasticsearch.xpack.security.authz.store;
 
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.health.ClusterIndexHealth;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.cache.Cache;
-import org.elasticsearch.common.cache.CacheBuilder;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.internal.Nullable;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.util.concurrent.ReleasableLock;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.xpack.common.IteratingActionListener;
-import org.elasticsearch.xpack.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.security.authz.RoleDescriptor.IndicesPrivileges;
-import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsCache;
-import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition;
-import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
-import org.elasticsearch.xpack.security.authz.permission.Role;
-import org.elasticsearch.xpack.security.authz.privilege.ClusterPrivilege;
-import org.elasticsearch.xpack.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.security.authz.privilege.Privilege;
-
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -48,6 +22,35 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.health.ClusterIndexHealth;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.internal.Nullable;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.ReleasableLock;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.common.IteratingActionListener;
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsCache;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
+import org.elasticsearch.xpack.security.authz.permission.Role;
+import org.elasticsearch.xpack.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.security.authz.privilege.Privilege;
+
 import static org.elasticsearch.xpack.security.Security.setting;
 
 /**
@@ -143,14 +146,21 @@ public class CompositeRolesStore extends AbstractComponent {
     }
 
     private void roleDescriptors(Set<String> roleNames, ActionListener<Set<RoleDescriptor>> roleDescriptorActionListener) {
-        final Set<String> filteredRoleNames =
-                roleNames.stream().filter((s) -> negativeLookupCache.contains(s) == false).collect(Collectors.toSet());
+        final Set<String> filteredRoleNames = roleNames.stream().filter((s) -> {
+            if (negativeLookupCache.contains(s)) {
+                logger.debug("Requested role [{}] does not exist (cached)", s);
+                return false;
+            } else {
+                return true;
+            }
+        }).collect(Collectors.toSet());
         final Set<RoleDescriptor> builtInRoleDescriptors = getBuiltInRoleDescriptors(filteredRoleNames);
         Set<String> remainingRoleNames = difference(filteredRoleNames, builtInRoleDescriptors);
         if (remainingRoleNames.isEmpty()) {
             roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
         } else {
             nativeRolesStore.getRoleDescriptors(remainingRoleNames.toArray(Strings.EMPTY_ARRAY), ActionListener.wrap((descriptors) -> {
+                logger.debug(() -> new ParameterizedMessage("Roles [{}] were resolved from the native index store", names(descriptors)));
                 builtInRoleDescriptors.addAll(descriptors);
                 callCustomRoleProvidersIfEnabled(builtInRoleDescriptors, filteredRoleNames, roleDescriptorActionListener);
             }, e -> {
@@ -169,6 +179,8 @@ public class CompositeRolesStore extends AbstractComponent {
                 new IteratingActionListener<>(roleDescriptorActionListener, (rolesProvider, listener) -> {
                     // resolve descriptors with role provider
                     rolesProvider.accept(missing, ActionListener.wrap((resolvedDescriptors) -> {
+                        logger.debug(() ->
+                                new ParameterizedMessage("Roles [{}] were resolved by [{}]", names(resolvedDescriptors), rolesProvider));
                         builtInRoleDescriptors.addAll(resolvedDescriptors);
                         // remove resolved descriptors from the set of roles still needed to be resolved
                         for (RoleDescriptor descriptor : resolvedDescriptors) {
@@ -187,6 +199,8 @@ public class CompositeRolesStore extends AbstractComponent {
                     return builtInRoleDescriptors;
                 }).run();
             } else {
+                logger.debug(() ->
+                        new ParameterizedMessage("Requested roles [{}] do not exist", Strings.collectionToCommaDelimitedString(missing)));
                 negativeLookupCache.addAll(missing);
                 roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
             }
@@ -199,15 +213,24 @@ public class CompositeRolesStore extends AbstractComponent {
         final Set<RoleDescriptor> descriptors = reservedRolesStore.roleDescriptors().stream()
                 .filter((rd) -> roleNames.contains(rd.getName()))
                 .collect(Collectors.toCollection(HashSet::new));
-
+        if (descriptors.size() > 0) {
+            logger.debug(() -> new ParameterizedMessage("Roles [{}] are builtin roles", names(descriptors)));
+        }
         final Set<String> difference = difference(roleNames, descriptors);
         if (difference.isEmpty() == false) {
-            descriptors.addAll(fileRolesStore.roleDescriptors(difference));
+            final Set<RoleDescriptor> fileRoles = fileRolesStore.roleDescriptors(difference);
+            logger.debug(() ->
+                    new ParameterizedMessage("Roles [{}] were resolved from [{}]", names(fileRoles), fileRolesStore.getFile()));
+            descriptors.addAll(fileRoles);
         }
 
         return descriptors;
     }
 
+    private String names(Collection<RoleDescriptor> descriptors) {
+        return descriptors.stream().map(RoleDescriptor::getName).collect(Collectors.joining(","));
+    }
+
     private Set<String> difference(Set<String> roleNames, Set<RoleDescriptor> descriptors) {
         Set<String> foundNames = descriptors.stream().map(RoleDescriptor::getName).collect(Collectors.toSet());
         return Sets.difference(roleNames, foundNames);
@@ -217,13 +240,12 @@ public class CompositeRolesStore extends AbstractComponent {
         if (roleDescriptors.isEmpty()) {
             return Role.EMPTY;
         }
-        StringBuilder nameBuilder = new StringBuilder();
         Set<String> clusterPrivileges = new HashSet<>();
         Set<String> runAs = new HashSet<>();
         Map<Set<String>, MergeableIndicesPrivilege> indicesPrivilegesMap = new HashMap<>();
+        List<String> roleNames = new ArrayList<>(roleDescriptors.size());
         for (RoleDescriptor descriptor : roleDescriptors) {
-            nameBuilder.append(descriptor.getName());
-            nameBuilder.append('_');
+            roleNames.add(descriptor.getName());
             if (descriptor.getClusterPrivileges() != null) {
                 clusterPrivileges.addAll(Arrays.asList(descriptor.getClusterPrivileges()));
             }
@@ -254,7 +276,7 @@ public class CompositeRolesStore extends AbstractComponent {
 
         final Set<String> clusterPrivs = clusterPrivileges.isEmpty() ? null : clusterPrivileges;
         final Privilege runAsPrivilege = runAs.isEmpty() ? Privilege.NONE : new Privilege(runAs, runAs.toArray(Strings.EMPTY_ARRAY));
-        Role.Builder builder = Role.builder(nameBuilder.toString(), fieldPermissionsCache)
+        Role.Builder builder = Role.builder(roleNames.toArray(new String[roleNames.size()]), fieldPermissionsCache)
                 .cluster(ClusterPrivilege.get(clusterPrivs))
                 .runAs(runAsPrivilege);
         indicesPrivilegesMap.entrySet().forEach((entry) -> {

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java

@@ -112,6 +112,10 @@ public class FileRolesStore extends AbstractComponent {
         }
     }
 
+    public Path getFile() {
+        return file;
+    }
+
     public static Path resolveFile(Environment env) {
         return XPackPlugin.resolveConfigFile(env, "roles.yml");
     }

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStore.java

@@ -5,26 +5,24 @@
  */
 package org.elasticsearch.xpack.security.authz.store;
 
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.xpack.monitoring.action.MonitoringBulkAction;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authz.permission.Role;
+import org.elasticsearch.xpack.security.SecurityExtension;
 import org.elasticsearch.xpack.security.support.MetadataUtils;
-import org.elasticsearch.xpack.security.user.KibanaUser;
 import org.elasticsearch.xpack.security.user.SystemUser;
 import org.elasticsearch.xpack.security.user.XPackUser;
-import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
-import org.elasticsearch.xpack.watcher.history.HistoryStore;
-import org.elasticsearch.xpack.watcher.watch.Watch;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.ServiceLoader;
 import java.util.Set;
 
 public class ReservedRolesStore {
 
-    public static final RoleDescriptor SUPERUSER_ROLE_DESCRIPTOR = new RoleDescriptor("superuser", new String[] { "all" },
+    public static final RoleDescriptor SUPERUSER_ROLE_DESCRIPTOR = new RoleDescriptor("superuser",
+            new String[] { "all" },
             new RoleDescriptor.IndicesPrivileges[] {
                     RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").build()},
             new String[] { "*" },
@@ -33,82 +31,16 @@ public class ReservedRolesStore {
     private static final Map<String, RoleDescriptor> RESERVED_ROLES = initializeReservedRoles();
 
     private static Map<String, RoleDescriptor> initializeReservedRoles() {
-        return MapBuilder.<String, RoleDescriptor>newMapBuilder()
-                .put("superuser", new RoleDescriptor("superuser", new String[] { "all" },
-                        new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").build()},
-                        new String[] { "*" },
-                        MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("transport_client", new RoleDescriptor("transport_client", new String[] { "transport_client" }, null, null,
-                        MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("kibana_user", new RoleDescriptor("kibana_user", null, new RoleDescriptor.IndicesPrivileges[] {
-                        RoleDescriptor.IndicesPrivileges.builder().indices(".kibana*").privileges("manage", "read", "index", "delete")
-                                .build() }, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("monitoring_user", new RoleDescriptor("monitoring_user", null, new RoleDescriptor.IndicesPrivileges[] {
-                        RoleDescriptor.IndicesPrivileges.builder()
-                                      .indices(".monitoring-*").privileges("read", "read_cross_cluster").build()
-                        },
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("remote_monitoring_agent", new RoleDescriptor("remote_monitoring_agent",
-                        new String[] {
-                            "manage_index_templates", "manage_ingest_pipelines", "monitor",
-                            "cluster:monitor/xpack/watcher/watch/get",
-                            "cluster:admin/xpack/watcher/watch/put",
-                            "cluster:admin/xpack/watcher/watch/delete",
-                        },
-                        new RoleDescriptor.IndicesPrivileges[] {
-                            RoleDescriptor.IndicesPrivileges.builder().indices(".monitoring-*").privileges("all").build() },
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("ingest_admin", new RoleDescriptor("ingest_admin", new String[] { "manage_index_templates", "manage_pipeline" },
-                        null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                 // reporting_user doesn't have any privileges in Elasticsearch, and Kibana authorizes privileges based on this role
-                .put("reporting_user", new RoleDescriptor("reporting_user", null, null,
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("kibana_dashboard_only_user", new RoleDescriptor(
-                        "kibana_dashboard_only_user",
-                        null,
-                        new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".kibana*").privileges("read", "view_index_metadata").build()
-                        },
-                        null,
-                        MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put(KibanaUser.ROLE_NAME, new RoleDescriptor(KibanaUser.ROLE_NAME,
-                        new String[] { "monitor", "manage_index_templates", MonitoringBulkAction.NAME },
-                        new RoleDescriptor.IndicesPrivileges[] {
-                            RoleDescriptor.IndicesPrivileges.builder().indices(".kibana*", ".reporting-*").privileges("all").build(),
-                            RoleDescriptor.IndicesPrivileges.builder()
-                                          .indices(".monitoring-*").privileges("read", "read_cross_cluster").build()
-                        },
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("logstash_system", new RoleDescriptor("logstash_system", new String[] { "monitor", MonitoringBulkAction.NAME},
-                        null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("machine_learning_user", new RoleDescriptor("machine_learning_user", new String[] { "monitor_ml" },
-                        new RoleDescriptor.IndicesPrivileges[] { RoleDescriptor.IndicesPrivileges.builder().indices(".ml-anomalies*",
-                                ".ml-notifications").privileges("view_index_metadata", "read").build() },
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("machine_learning_admin", new RoleDescriptor("machine_learning_admin", new String[] { "manage_ml" },
-                        new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder().indices(".ml-*").privileges("view_index_metadata", "read")
-                                        .build() }, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("watcher_admin", new RoleDescriptor("watcher_admin", new String[] { "manage_watcher" },
-                        new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder().indices(Watch.INDEX, TriggeredWatchStore.INDEX_NAME,
-                                        HistoryStore.INDEX_PREFIX + "*").privileges("read").build() },
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("watcher_user", new RoleDescriptor("watcher_user", new String[] { "monitor_watcher" },
-                        new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder().indices(Watch.INDEX)
-                                        .privileges("read")
-                                        .build(),
-                                RoleDescriptor.IndicesPrivileges.builder().indices(HistoryStore.INDEX_PREFIX + "*")
-                                        .privileges("read")
-                                        .build() }, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .put("logstash_admin", new RoleDescriptor("logstash_admin", null, new RoleDescriptor.IndicesPrivileges[] {
-                    RoleDescriptor.IndicesPrivileges.builder().indices(".logstash*")
-                        .privileges("create", "delete", "index", "manage", "read").build() },
-                    null, MetadataUtils.DEFAULT_RESERVED_METADATA))
-                .immutableMap();
+        Map<String, RoleDescriptor> roles = new HashMap<>();
+
+        roles.put("superuser", SUPERUSER_ROLE_DESCRIPTOR);
+
+        // Services are loaded through SPI, and are defined in their META-INF/services
+        for(SecurityExtension ext : ServiceLoader.load(SecurityExtension.class, SecurityExtension.class.getClassLoader())) {
+            roles.putAll(ext.getReservedRoles());
+        }
+
+        return Collections.unmodifiableMap(roles);
     }
 
     public Map<String, Object> usageStats() {

plugin/src/main/java/org/elasticsearch/xpack/security/user/ElasticUser.java

@@ -7,8 +7,6 @@ package org.elasticsearch.xpack.security.user;
 
 import org.elasticsearch.xpack.security.support.MetadataUtils;
 
-import java.util.HashMap;
-import java.util.Map;
 
 /**
  * The reserved {@code elastic} superuser. Has full permission/access to the cluster/indices and can
@@ -17,7 +15,8 @@ import java.util.Map;
 public class ElasticUser extends User {
 
     public static final String NAME = "elastic";
-    private static final String ROLE_NAME = "superuser";
+    // used for testing in a different package
+    public static final String ROLE_NAME = "superuser";
 
     public ElasticUser(boolean enabled) {
         super(NAME, new String[] { ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled);

plugin/src/main/java/org/elasticsearch/xpack/security/user/LogstashSystemUser.java

@@ -14,7 +14,7 @@ import org.elasticsearch.xpack.security.support.MetadataUtils;
 public class LogstashSystemUser extends User {
 
     public static final String NAME = "logstash_system";
-    private static final String ROLE_NAME = "logstash_system";
+    public static final String ROLE_NAME = "logstash_system";
     public static final Version DEFINED_SINCE = Version.V_5_2_0;
     public static final BuiltinUserInfo USER_INFO = new BuiltinUserInfo(NAME, ROLE_NAME, DEFINED_SINCE);
 

plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -449,7 +449,7 @@ public class Watcher implements ActionPlugin {
                     new FixedExecutorBuilder(
                             settings,
                             InternalWatchExecutor.THREAD_POOL_NAME,
-                            5 * EsExecutors.numberOfProcessors(settings),
+                            getWatcherThreadPoolSize(settings),
                             1000,
                             "xpack.watcher.thread_pool");
             return Collections.singletonList(builder);
@@ -457,6 +457,28 @@ public class Watcher implements ActionPlugin {
         return Collections.emptyList();
     }
 
+    /**
+     * A method to indicate the size of the watcher thread pool
+     * As watches are primarily bound on I/O waiting and execute
+     * synchronously, it makes sense to have a certain minimum of a
+     * threadpool size. This means you should start with a fair number
+     * of threads which is more than the number of CPUs, but you also need
+     * to ensure that this number does not go crazy high if you have really
+     * beefy machines. This can still be configured manually.
+     *
+     * Calculation is as follows:
+     * Use five times the number of processors up until 50, then stick with the
+     * number of processors.
+     *
+     * @param settings The current settings
+     * @return A number between 5 and the number of processors
+     */
+    static int getWatcherThreadPoolSize(Settings settings) {
+        int numberOfProcessors = EsExecutors.numberOfProcessors(settings);
+        long size = Math.max(Math.min(5 * numberOfProcessors, 50), numberOfProcessors);
+        return Math.toIntExact(size);
+    }
+
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         if (false == enabled) {

plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherSecurityExtension.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher;
+
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.SecurityExtension;
+import org.elasticsearch.xpack.security.support.MetadataUtils;
+import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
+import org.elasticsearch.xpack.watcher.history.HistoryStore;
+import org.elasticsearch.xpack.watcher.watch.Watch;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class WatcherSecurityExtension implements SecurityExtension {
+    @Override
+    public Map<String, RoleDescriptor> getReservedRoles() {
+        Map<String, RoleDescriptor> roles = new HashMap<>();
+
+        roles.put("watcher_admin",
+                  new RoleDescriptor("watcher_admin",
+                                     new String[] { "manage_watcher" },
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(Watch.INDEX,
+                                                          TriggeredWatchStore.INDEX_NAME,
+                                                          HistoryStore.INDEX_PREFIX + "*")
+                                                 .privileges("read").build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        roles.put("watcher_user",
+                  new RoleDescriptor("watcher_user",
+                                     new String[] { "monitor_watcher" },
+                                     new RoleDescriptor.IndicesPrivileges[] {
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(Watch.INDEX)
+                                                 .privileges("read")
+                                                 .build(),
+                                         RoleDescriptor.IndicesPrivileges.builder()
+                                                 .indices(HistoryStore.INDEX_PREFIX + "*")
+                                                 .privileges("read")
+                                                 .build()
+                                     },
+                                     null,
+                                     MetadataUtils.DEFAULT_RESERVED_METADATA));
+
+        return Collections.unmodifiableMap(roles);
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/IncidentEvent.java

@@ -121,7 +121,7 @@ public class IncidentEvent implements ToXContentObject {
                             builder.endObject();
                         }
                         if (contexts != null && contexts.length > 0) {
-                            builder.startArray(Fields.CONTEXT.getPreferredName());
+                            builder.startArray(Fields.CONTEXTS.getPreferredName());
                             for (IncidentEventContext context : contexts) {
                                 context.toXContent(builder, params);
                             }
@@ -154,7 +154,7 @@ public class IncidentEvent implements ToXContentObject {
         }
         builder.field(Fields.ATTACH_PAYLOAD.getPreferredName(), attachPayload);
         if (contexts != null) {
-            builder.startArray(Fields.CONTEXT.getPreferredName());
+            builder.startArray(Fields.CONTEXTS.getPreferredName());
             for (IncidentEventContext context : contexts) {
                 context.toXContent(builder, params);
             }
@@ -265,7 +265,7 @@ public class IncidentEvent implements ToXContentObject {
                 proxy.toXContent(builder, params);
             }
             if (contexts != null) {
-                builder.startArray(Fields.CONTEXT.getPreferredName());
+                builder.startArray(Fields.CONTEXTS.getPreferredName());
                 for (IncidentEventContext.Template context : contexts) {
                     context.toXContent(builder, params);
                 }
@@ -341,7 +341,7 @@ public class IncidentEvent implements ToXContentObject {
                         throw new ElasticsearchParseException("could not parse pager duty event template. failed to parse field [{}], " +
                                 "expected a boolean value but found [{}] instead", Fields.ATTACH_PAYLOAD.getPreferredName(), token);
                     }
-                } else if (Fields.CONTEXT.match(currentFieldName)) {
+                } else if (Fields.CONTEXTS.match(currentFieldName) || Fields.CONTEXT_DEPRECATED.match(currentFieldName)) {
                     if (token == XContentParser.Token.START_ARRAY) {
                         List<IncidentEventContext.Template> list = new ArrayList<>();
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
@@ -349,7 +349,7 @@ public class IncidentEvent implements ToXContentObject {
                                 list.add(IncidentEventContext.Template.parse(parser));
                             } catch (ElasticsearchParseException e) {
                                 throw new ElasticsearchParseException("could not parse pager duty event template. failed to parse field " +
-                                        "[{}]", Fields.CONTEXT.getPreferredName());
+                                        "[{}]", parser.currentName());
                             }
                         }
                         contexts = list.toArray(new IncidentEventContext.Template[list.size()]);
@@ -438,7 +438,11 @@ public class IncidentEvent implements ToXContentObject {
         ParseField CLIENT = new ParseField("client");
         ParseField CLIENT_URL = new ParseField("client_url");
         ParseField ATTACH_PAYLOAD = new ParseField("attach_payload");
-        ParseField CONTEXT = new ParseField("context");
+        ParseField CONTEXTS = new ParseField("contexts");
+        // this field exists because in versions prior 6.0 we accidentally used context instead of contexts and thus the correct data
+        // was never picked up on the pagerduty side
+        // we need to keep this for BWC
+        ParseField CONTEXT_DEPRECATED = new ParseField("context");
 
         ParseField SERVICE_KEY = new ParseField("service_key");
         ParseField PAYLOAD = new ParseField("payload");

plugin/src/main/resources/META-INF/services/org.elasticsearch.xpack.security.SecurityExtension

@@ -0,0 +1,5 @@
+org.elasticsearch.xpack.logstash.LogstashSecurityExtension
+org.elasticsearch.xpack.ml.MachineLearningSecurityExtension
+org.elasticsearch.xpack.monitoring.MonitoringSecurityExtension
+org.elasticsearch.xpack.security.StackSecurityExtension
+org.elasticsearch.xpack.watcher.WatcherSecurityExtension

plugin/src/main/resources/monitoring-beats.json

@@ -37,10 +37,6 @@
             },
             "name": {
               "type": "keyword"
-            },
-            "timestamp": {
-              "type": "date",
-              "format": "date_time"
             }
           }
         },
@@ -71,6 +67,136 @@
                   "type": "keyword"
                 }
               }
+            },
+            "metrics": {
+              "properties": {
+                "beat": {
+                  "properties": {
+                    "memstats": {
+                      "properties": {
+                        "gc_next": {
+                          "type": "long"
+                        },
+                        "memory_alloc": {
+                          "type": "long"
+                        },
+                        "memory_total": {
+                          "type": "long"
+                        }
+                      }
+                    }
+                  }
+                },
+                "libbeat": {
+                  "properties": {
+                    "config": {
+                      "properties": {
+                        "module": {
+                          "properties": {
+                            "running": {
+                              "type": "long"
+                            },
+                            "starts": {
+                              "type": "long"
+                            },
+                            "stops": {
+                              "type": "long"
+                            }
+                          }
+                        },
+                        "reloads": {
+                          "type": "long"
+                        }
+                      }
+                    },
+                    "output": {
+                      "properties": {
+                        "events": {
+                          "properties": {
+                            "acked": {
+                              "type": "long"
+                            },
+                            "active": {
+                              "type": "long"
+                            },
+                            "batches": {
+                              "type": "long"
+                            },
+                            "failed": {
+                              "type": "long"
+                            },
+                            "total": {
+                              "type": "long"
+                            }
+                          }
+                        },
+                        "read": {
+                          "properties": {
+                            "bytes": {
+                              "type": "long"
+                            },
+                            "errors": {
+                              "type": "long"
+                            }
+                          }
+                        },
+                        "type": {
+                          "type": "keyword"
+                        },
+                        "write": {
+                          "properties": {
+                            "bytes": {
+                              "type": "long"
+                            },
+                            "errors": {
+                              "type": "long"
+                            }
+                          }
+                        }
+                      }
+                    },
+                    "pipeline": {
+                      "properties": {
+                        "clients": {
+                          "type": "long"
+                        },
+                        "events": {
+                          "properties": {
+                            "active": {
+                              "type": "long"
+                            },
+                            "dropped": {
+                              "type": "long"
+                            },
+                            "failed": {
+                              "type": "long"
+                            },
+                            "filtered": {
+                              "type": "long"
+                            },
+                            "published": {
+                              "type": "long"
+                            },
+                            "retry": {
+                              "type": "long"
+                            },
+                            "total": {
+                              "type": "long"
+                            }
+                          }
+                        },
+                        "queue": {
+                          "properties": {
+                            "acked": {
+                              "type": "long"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
             }
           }
         }

plugin/src/main/resources/security_audit_log.json

@@ -40,6 +40,9 @@
         "principal": {
           "type": "keyword"
         },
+        "roles": {
+          "type": "keyword"
+        },
         "run_by_principal": {
           "type": "keyword"
         },

plugin/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java

@@ -5,24 +5,36 @@
  */
 package org.elasticsearch.integration;
 
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
-import org.elasticsearch.test.SecurityIntegTestCase;
 
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
-import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.equalTo;
 
 public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
@@ -92,7 +104,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
                 .build();
     }
 
-    public void testSimpleQuery() throws Exception {
+    public void testSimpleQuery() {
         assertAcked(client().admin().indices().prepareCreate("test")
                         .addMapping("type1", "field1", "type=text", "field2", "type=text")
         );
@@ -130,7 +142,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(response.getHits().getAt(1).getSourceAsMap().get("field2").toString(), equalTo("value2"));
     }
 
-    public void testDLSIsAppliedBeforeFLS() throws Exception {
+    public void testDLSIsAppliedBeforeFLS() {
         assertAcked(client().admin().indices().prepareCreate("test")
                 .addMapping("type1", "field1", "type=text", "field2", "type=text")
         );
@@ -157,7 +169,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
         assertHitCount(response, 0);
     }
 
-    public void testQueryCache() throws Exception {
+    public void testQueryCache() {
         assertAcked(client().admin().indices().prepareCreate("test")
                         .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))
                         .addMapping("type1", "field1", "type=text", "field2", "type=text")
@@ -214,4 +226,216 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
         }
     }
 
+    public void testGetMappingsIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field1");
+        }
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field2");
+        }
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field1");
+        }
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field1", "field2");
+        }
+    }
+
+    public void testGetIndexMappingsIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field1");
+        }
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field2");
+        }
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field1");
+        }
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field1", "field2");
+        }
+    }
+
+    public void testGetFieldMappingsIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field1");
+        }
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field2");
+        }
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field1");
+        }
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field1", "field2");
+        }
+    }
+
+    public void testFieldCapabilitiesIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field1");
+        }
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field2");
+        }
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field1");
+        }
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field1", "field2");
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertExpectedFields(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings,
+                                             String... fields) {
+        Map<String, Object> sourceAsMap = mappings.get("test").get("type1").getSourceAsMap();
+        assertEquals(1, sourceAsMap.size());
+        Map<String, Object> properties = (Map<String, Object>)sourceAsMap.get("properties");
+        assertEquals(fields.length, properties.size());
+        for (String field : fields) {
+            assertNotNull(properties.get(field));
+        }
+    }
+
+    private static void assertExpectedFields(FieldCapabilitiesResponse fieldCapabilitiesResponse, String... expectedFields) {
+        Map<String, Map<String, FieldCapabilities>> responseMap = fieldCapabilitiesResponse.get();
+        Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
+        for (String field : builtInMetaDataFields) {
+            Map<String, FieldCapabilities> remove = responseMap.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", remove);
+        }
+        for (String field : expectedFields) {
+            Map<String, FieldCapabilities> remove = responseMap.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", remove);
+        }
+        assertEquals("Some unexpected fields were returned: " + responseMap.keySet(), 0, responseMap.size());
+    }
+
+    private static void assertExpectedFields(Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>> mappings,
+                                            String... expectedFields) {
+        assertEquals(1, mappings.size());
+        Map<String, GetFieldMappingsResponse.FieldMappingMetaData> fields = new HashMap<>(mappings.get("type1"));
+        Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
+        for (String field : builtInMetaDataFields) {
+            GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", fieldMappingMetaData);
+        }
+        for (String field : expectedFields) {
+            GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field);
+            assertNotNull("expected field [" + field + "] not found", fieldMappingMetaData);
+        }
+        assertEquals("Some unexpected fields were returned: " + fields.keySet(), 0, fields.size());
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionTests.java

@@ -192,7 +192,7 @@ public class OpenJobActionTests extends ESTestCase {
         metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
         cs.metaData(metaData);
         cs.routingTable(routingTable.build());
-        Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id2", cs.build(), 2, maxRunningJobsPerNode, 30, logger);
+        Assignment result = OpenJobAction.selectLeastLoadedMlNode("job_id0", cs.build(), 2, maxRunningJobsPerNode, 30, logger);
         assertNull(result.getExecutorNode());
         assertTrue(result.getExplanation().contains("because this node is full. Number of opened jobs [" + maxRunningJobsPerNode
                 + "], xpack.ml.max_open_jobs [" + maxRunningJobsPerNode + "]"));

plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigTests.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.ml.datafeed;
 
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
-
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -33,7 +32,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ml.datafeed.ChunkingConfig.Mode;
-import org.elasticsearch.xpack.ml.job.config.JobTests;
 import org.elasticsearch.xpack.ml.job.messages.Messages;
 import org.joda.time.DateTimeZone;
 
@@ -79,23 +77,29 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
             }
             builder.setScriptFields(scriptFields);
         }
+        Long aggHistogramInterval = null;
         if (randomBoolean() && addScriptFields == false) {
             // can only test with a single agg as the xcontent order gets randomized by test base class and then
             // the actual xcontent isn't the same and test fail.
             // Testing with a single agg is ok as we don't have special list writeable / xconent logic
             AggregatorFactories.Builder aggs = new AggregatorFactories.Builder();
-            long interval = randomNonNegativeLong();
-            interval = interval > bucketSpanMillis ? bucketSpanMillis : interval;
-            interval = interval <= 0 ? 1 : interval;
+            aggHistogramInterval = randomNonNegativeLong();
+            aggHistogramInterval = aggHistogramInterval> bucketSpanMillis ? bucketSpanMillis : aggHistogramInterval;
+            aggHistogramInterval = aggHistogramInterval <= 0 ? 1 : aggHistogramInterval;
             MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
-            aggs.addAggregator(AggregationBuilders.dateHistogram("buckets").interval(interval).subAggregation(maxTime).field("time"));
+            aggs.addAggregator(AggregationBuilders.dateHistogram("buckets")
+                    .interval(aggHistogramInterval).subAggregation(maxTime).field("time"));
             builder.setAggregations(aggs);
         }
         if (randomBoolean()) {
             builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE));
         }
         if (randomBoolean()) {
-            builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
+            if (aggHistogramInterval == null) {
+                builder.setFrequency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
+            } else {
+                builder.setFrequency(TimeValue.timeValueMillis(randomIntBetween(1, 5) * aggHistogramInterval));
+            }
         }
         if (randomBoolean()) {
             builder.setQueryDelay(TimeValue.timeValueMillis(randomIntBetween(1, 1_000_000)));
@@ -398,6 +402,90 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
         assertEquals("Aggregations can only have 1 date_histogram or histogram aggregation", e.getMessage());
     }
 
+    public void testDefaultFrequency_GivenNegative() {
+        DatafeedConfig datafeed = createTestInstance();
+        ESTestCase.expectThrows(IllegalArgumentException.class, () -> datafeed.defaultFrequency(TimeValue.timeValueSeconds(-1)));
+    }
+
+    public void testDefaultFrequency_GivenNoAggregations() {
+        DatafeedConfig.Builder datafeedBuilder = new DatafeedConfig.Builder("feed", "job");
+        datafeedBuilder.setIndices(Arrays.asList("my_index"));
+        DatafeedConfig datafeed = datafeedBuilder.build();
+
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(30)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(60)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(90)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(120)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(121)));
+
+        assertEquals(TimeValue.timeValueSeconds(61), datafeed.defaultFrequency(TimeValue.timeValueSeconds(122)));
+        assertEquals(TimeValue.timeValueSeconds(75), datafeed.defaultFrequency(TimeValue.timeValueSeconds(150)));
+        assertEquals(TimeValue.timeValueSeconds(150), datafeed.defaultFrequency(TimeValue.timeValueSeconds(300)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1200)));
+
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1201)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1800)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(1)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(2)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(12)));
+
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(12 * 3600 + 1)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(13)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(24)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(48)));
+    }
+
+    public void testDefaultFrequency_GivenAggregationsWithHistogramInterval_1_Second() {
+        DatafeedConfig datafeed = createDatafeedWithDateHistogram("1s");
+
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(60)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(90)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(120)));
+        assertEquals(TimeValue.timeValueSeconds(125), datafeed.defaultFrequency(TimeValue.timeValueSeconds(250)));
+        assertEquals(TimeValue.timeValueSeconds(250), datafeed.defaultFrequency(TimeValue.timeValueSeconds(500)));
+
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(1)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(13)));
+    }
+
+    public void testDefaultFrequency_GivenAggregationsWithHistogramInterval_1_Minute() {
+        DatafeedConfig datafeed = createDatafeedWithDateHistogram("1m");
+
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(60)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(90)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(120)));
+        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(180)));
+        assertEquals(TimeValue.timeValueMinutes(2), datafeed.defaultFrequency(TimeValue.timeValueSeconds(240)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(20)));
+
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueSeconds(20 * 60 + 1)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(6)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(12)));
+
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(13)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(72)));
+    }
+
+    public void testDefaultFrequency_GivenAggregationsWithHistogramInterval_10_Minutes() {
+        DatafeedConfig datafeed = createDatafeedWithDateHistogram("10m");
+
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(10)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(20)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(30)));
+        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(12 * 60)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueMinutes(13 * 60)));
+    }
+
+    public void testDefaultFrequency_GivenAggregationsWithHistogramInterval_1_Hour() {
+        DatafeedConfig datafeed = createDatafeedWithDateHistogram("1h");
+
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(1)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(3601)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(2)));
+        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(12)));
+    }
+
     public static String randomValidDatafeedId() {
         CodepointSetGenerator generator =  new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
         return generator.ofCodePointsLength(random(), 10, 10);

plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java

@@ -141,6 +141,39 @@ public class DatafeedJobValidatorTests extends ESTestCase {
         DatafeedJobValidator.validate(goodDatafeedConfig, job);
     }
 
+    public void testVerify_FrequencyIsMultipleOfHistogramInterval() throws IOException {
+        Job.Builder builder = buildJobBuilder("foo");
+        AnalysisConfig.Builder ac = createAnalysisConfig();
+        ac.setSummaryCountFieldName("some_count");
+        ac.setBucketSpan(TimeValue.timeValueMinutes(5));
+        builder.setAnalysisConfig(ac);
+        Job job = builder.build(new Date());
+        DatafeedConfig.Builder datafeedBuilder = createValidDatafeedConfigWithAggs(60 * 1000);
+
+        // Check with multiples
+        datafeedBuilder.setFrequency(TimeValue.timeValueSeconds(60));
+        DatafeedJobValidator.validate(datafeedBuilder.build(), job);
+        datafeedBuilder.setFrequency(TimeValue.timeValueSeconds(120));
+        DatafeedJobValidator.validate(datafeedBuilder.build(), job);
+        datafeedBuilder.setFrequency(TimeValue.timeValueSeconds(180));
+        DatafeedJobValidator.validate(datafeedBuilder.build(), job);
+        datafeedBuilder.setFrequency(TimeValue.timeValueSeconds(240));
+        DatafeedJobValidator.validate(datafeedBuilder.build(), job);
+        datafeedBuilder.setFrequency(TimeValue.timeValueHours(1));
+        DatafeedJobValidator.validate(datafeedBuilder.build(), job);
+
+        // Now non-multiples
+        datafeedBuilder.setFrequency(TimeValue.timeValueSeconds(30));
+        ElasticsearchStatusException e = ESTestCase.expectThrows(ElasticsearchStatusException.class,
+                () -> DatafeedJobValidator.validate(datafeedBuilder.build(), job));
+        assertEquals("Datafeed frequency [30s] must be a multiple of the aggregation interval [60000ms]", e.getMessage());
+
+        datafeedBuilder.setFrequency(TimeValue.timeValueSeconds(90));
+        e = ESTestCase.expectThrows(ElasticsearchStatusException.class,
+                () -> DatafeedJobValidator.validate(datafeedBuilder.build(), job));
+        assertEquals("Datafeed frequency [1.5m] must be a multiple of the aggregation interval [60000ms]", e.getMessage());
+    }
+
     private static Job.Builder buildJobBuilder(String id) {
         Job.Builder builder = new Job.Builder(id);
         AnalysisConfig.Builder ac = createAnalysisConfig();

plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/DefaultFrequencyTests.java

@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.datafeed;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.time.Duration;
-
-public class DefaultFrequencyTests extends ESTestCase {
-
-    public void testCalc_GivenNegative() {
-        ESTestCase.expectThrows(IllegalArgumentException.class, () -> DefaultFrequency.ofBucketSpan(-1));
-    }
-
-
-    public void testCalc() {
-        assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(1));
-        assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(30));
-        assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(60));
-        assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(90));
-        assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(120));
-        assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(121));
-
-        assertEquals(Duration.ofSeconds(61), DefaultFrequency.ofBucketSpan(122));
-        assertEquals(Duration.ofSeconds(75), DefaultFrequency.ofBucketSpan(150));
-        assertEquals(Duration.ofSeconds(150), DefaultFrequency.ofBucketSpan(300));
-        assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1200));
-
-        assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1201));
-        assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1800));
-        assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(3600));
-        assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(7200));
-        assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(12 * 3600));
-
-        assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(12 * 3600 + 1));
-        assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(13 * 3600));
-        assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(24 * 3600));
-        assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(48 * 3600));
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java

@@ -5,6 +5,10 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor.scroll;
 
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -21,6 +25,7 @@ import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
+import org.mockito.ArgumentCaptor;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -42,6 +47,7 @@ import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.Matchers.same;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -50,7 +56,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
     private Client client;
     private List<SearchRequestBuilder> capturedSearchRequests;
     private List<String> capturedContinueScrollIds;
-    private List<String> capturedClearScrollIds;
+    private ArgumentCaptor<ClearScrollRequest> capturedClearScrollRequests;
     private String jobId;
     private ExtractedFields extractedFields;
     private List<String> types;
@@ -59,6 +65,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
     private List<SearchSourceBuilder.ScriptField> scriptFields;
     private int scrollSize;
     private long initScrollStartTime;
+    private ActionFuture<ClearScrollResponse> clearScrollFuture;
 
     private class TestDataExtractor extends ScrollDataExtractor {
 
@@ -95,11 +102,6 @@ public class ScrollDataExtractorTests extends ESTestCase {
             }
         }
 
-        @Override
-        void clearScroll(String scrollId) {
-            capturedClearScrollIds.add(scrollId);
-        }
-
         void setNextResponse(SearchResponse searchResponse) {
             responses.add(searchResponse);
         }
@@ -118,7 +120,6 @@ public class ScrollDataExtractorTests extends ESTestCase {
         client = mock(Client.class);
         capturedSearchRequests = new ArrayList<>();
         capturedContinueScrollIds = new ArrayList<>();
-        capturedClearScrollIds = new ArrayList<>();
         jobId = "test-job";
         ExtractedField timeField = ExtractedField.newField("time", ExtractedField.ExtractionMethod.DOC_VALUE);
         extractedFields = new ExtractedFields(timeField,
@@ -128,6 +129,10 @@ public class ScrollDataExtractorTests extends ESTestCase {
         query = QueryBuilders.matchAllQuery();
         scriptFields = Collections.emptyList();
         scrollSize = 1000;
+
+        clearScrollFuture = mock(ActionFuture.class);
+        capturedClearScrollRequests = ArgumentCaptor.forClass(ClearScrollRequest.class);
+        when(client.execute(same(ClearScrollAction.INSTANCE), capturedClearScrollRequests.capture())).thenReturn(clearScrollFuture);
     }
 
     public void testSinglePageExtraction() throws IOException {
@@ -164,6 +169,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
         assertThat(capturedContinueScrollIds.size(), equalTo(1));
         assertThat(capturedContinueScrollIds.get(0), equalTo(response1.getScrollId()));
 
+        List<String> capturedClearScrollIds = getCapturedClearScrollIds();
         assertThat(capturedClearScrollIds.size(), equalTo(1));
         assertThat(capturedClearScrollIds.get(0), equalTo(response2.getScrollId()));
     }
@@ -215,6 +221,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
         assertThat(capturedContinueScrollIds.get(0), equalTo(response1.getScrollId()));
         assertThat(capturedContinueScrollIds.get(1), equalTo(response2.getScrollId()));
 
+        List<String> capturedClearScrollIds = getCapturedClearScrollIds();
         assertThat(capturedClearScrollIds.size(), equalTo(1));
         assertThat(capturedClearScrollIds.get(0), equalTo(response3.getScrollId()));
     }
@@ -252,6 +259,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
         assertThat(asString(stream.get()), equalTo(expectedStream));
         assertThat(extractor.hasNext(), is(false));
 
+        List<String> capturedClearScrollIds = getCapturedClearScrollIds();
         assertThat(capturedClearScrollIds.size(), equalTo(1));
         assertThat(capturedClearScrollIds.get(0), equalTo(response2.getScrollId()));
     }
@@ -392,6 +400,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
 
         expectThrows(IOException.class, () -> extractor.next());
 
+        List<String> capturedClearScrollIds = getCapturedClearScrollIds();
         assertThat(capturedClearScrollIds.isEmpty(), is(true));
     }
 
@@ -445,6 +454,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
         assertThat(capturedContinueScrollIds.size(), equalTo(1));
         assertThat(capturedContinueScrollIds.get(0), equalTo(response1.getScrollId()));
 
+        List<String> capturedClearScrollIds = getCapturedClearScrollIds();
         assertThat(capturedClearScrollIds.size(), equalTo(1));
         assertThat(capturedClearScrollIds.get(0), equalTo(response2.getScrollId()));
     }
@@ -500,6 +510,10 @@ public class ScrollDataExtractorTests extends ESTestCase {
         return searchResponse;
     }
 
+    private List<String> getCapturedClearScrollIds() {
+        return capturedClearScrollRequests.getAllValues().stream().map(r -> r.getScrollIds().get(0)).collect(Collectors.toList());
+    }
+
     private static String asString(InputStream inputStream) throws IOException {
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
             return reader.lines().collect(Collectors.joining("\n"));

plugin/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java

@@ -96,7 +96,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
         DatafeedConfig.Builder configBuilder = createDatafeedBuilder("data_feed_id", job.getId(), Collections.singletonList("*"));
 
         MaxAggregationBuilder maxAggregation = AggregationBuilders.max("time").field("time");
-        HistogramAggregationBuilder histogramAggregation = AggregationBuilders.histogram("time").interval(300000)
+        HistogramAggregationBuilder histogramAggregation = AggregationBuilders.histogram("time").interval(60000)
                 .subAggregation(maxAggregation).field("time");
 
         configBuilder.setAggregations(AggregatorFactories.builder().addAggregator(histogramAggregation));
@@ -216,7 +216,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
 
             DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
             assertThat(node.getAttributes(), hasEntry(MachineLearning.ML_ENABLED_NODE_ATTR, "true"));
-            assertThat(node.getAttributes(), hasEntry(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10"));
+            assertThat(node.getAttributes(), hasEntry(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "20"));
             JobTaskStatus jobTaskStatus = (JobTaskStatus) task.getStatus();
             assertNotNull(jobTaskStatus);
             assertEquals(JobState.OPENED, jobTaskStatus.getState());
@@ -402,7 +402,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
             assertFalse(task.needsReassignment(clusterState.nodes()));
             DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
             assertThat(node.getAttributes(), hasEntry(MachineLearning.ML_ENABLED_NODE_ATTR, "true"));
-            assertThat(node.getAttributes(), hasEntry(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10"));
+            assertThat(node.getAttributes(), hasEntry(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "20"));
 
             JobTaskStatus jobTaskStatus = (JobTaskStatus) task.getStatus();
             assertNotNull(jobTaskStatus);

plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java

@@ -9,6 +9,7 @@ import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
 import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction;
@@ -17,6 +18,7 @@ import org.elasticsearch.xpack.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.ml.action.KillProcessAction;
 import org.elasticsearch.xpack.ml.action.PutJobAction;
 import org.elasticsearch.xpack.ml.action.StopDatafeedAction;
+import org.elasticsearch.xpack.ml.datafeed.ChunkingConfig;
 import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.ml.job.config.Job;
@@ -32,10 +34,12 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed;
+import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeedBuilder;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.getDataCounts;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.indexDocs;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase {
 
@@ -223,6 +227,59 @@ public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase {
         });
     }
 
+    /**
+     * Stopping a lookback closes the associated job _after_ the stop call returns.
+     * This test ensures that a kill request submitted during this close doesn't
+     * put the job into the "failed" state.
+     */
+    public void testStopLookbackFollowedByProcessKill() throws Exception {
+        client().admin().indices().prepareCreate("data")
+                .addMapping("type", "time", "type=date")
+                .get();
+        long numDocs = randomIntBetween(1024, 2048);
+        long now = System.currentTimeMillis();
+        long oneWeekAgo = now - 604800000;
+        long twoWeeksAgo = oneWeekAgo - 604800000;
+        indexDocs(logger, "data", numDocs, twoWeeksAgo, oneWeekAgo);
+
+        Job.Builder job = createScheduledJob("lookback-job-stopped-then-killed");
+        registerJob(job);
+        PutJobAction.Response putJobResponse = putJob(job);
+        assertTrue(putJobResponse.isAcknowledged());
+        assertThat(putJobResponse.getResponse().getJobVersion(), equalTo(Version.CURRENT));
+        openJob(job.getId());
+        assertBusy(() -> assertEquals(getJobStats(job.getId()).get(0).getState(), JobState.OPENED));
+
+        List<String> t = Collections.singletonList("data");
+        DatafeedConfig.Builder datafeedConfigBuilder = createDatafeedBuilder(job.getId() + "-datafeed", job.getId(), t);
+        // Use lots of chunks so we have time to stop the lookback before it completes
+        datafeedConfigBuilder.setChunkingConfig(ChunkingConfig.newManual(new TimeValue(1, TimeUnit.SECONDS)));
+        DatafeedConfig datafeedConfig = datafeedConfigBuilder.build();
+        registerDatafeed(datafeedConfig);
+        assertTrue(putDatafeed(datafeedConfig).isAcknowledged());
+
+        startDatafeed(datafeedConfig.getId(), 0L, now);
+        assertBusy(() -> {
+            DataCounts dataCounts = getDataCounts(job.getId());
+            assertThat(dataCounts.getProcessedRecordCount(), greaterThan(0L));
+        }, 60, TimeUnit.SECONDS);
+
+        stopDatafeed(datafeedConfig.getId());
+
+        // At this point, stopping the datafeed will have submitted a request for the job to close.
+        // Depending on thread scheduling, the following kill request might overtake it.  The Thread.sleep()
+        // call here makes it more likely; to make it inevitable for testing also add a Thread.sleep(10)
+        // immediately before the checkProcessIsAlive() call in AutodetectCommunicator.close().
+        Thread.sleep(randomIntBetween(1, 9));
+
+        KillProcessAction.Request killRequest = new KillProcessAction.Request(job.getId());
+        client().execute(KillProcessAction.INSTANCE, killRequest).actionGet();
+
+        // This should close very quickly, as we killed the process.  If the job goes into the "failed"
+        // state that's wrong and this test will fail.
+        waitUntilJobIsClosed(job.getId(), TimeValue.timeValueSeconds(2));
+    }
+
     private void startRealtime(String jobId) throws Exception {
         client().admin().indices().prepareCreate("data")
                 .addMapping("type", "time", "type=date")

plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java

@@ -16,6 +16,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
 import org.junit.After;
 import org.junit.Before;
 
@@ -720,6 +721,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
     @After
     public void clearMlState() throws Exception {
         new MlRestTestStateCleaner(logger, adminClient(), this).clearMlMetadata();
+        XPackRestTestHelper.waitForPendingTasks(adminClient());
     }
 
     private static class DatafeedBuilder {

plugin/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.integration;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.ml.job.config.DataDescription;
@@ -134,6 +135,27 @@ public class ForecastIT extends MlNativeAutodetectIntegTestCase {
         }
     }
 
+    public void testDurationCannotBeLessThanBucketSpan() throws Exception {
+        Detector.Builder detector = new Detector.Builder("mean", "value");
+
+        TimeValue bucketSpan = TimeValue.timeValueHours(1);
+        AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
+        analysisConfig.setBucketSpan(bucketSpan);
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeFormat("epoch");
+        Job.Builder job = new Job.Builder("forecast-it-test-duration-bucket-span");
+        job.setAnalysisConfig(analysisConfig);
+        job.setDataDescription(dataDescription);
+
+        registerJob(job);
+        putJob(job);
+        openJob(job.getId());
+        ElasticsearchException e = expectThrows(ElasticsearchException.class,() -> forecast(job.getId(),
+                TimeValue.timeValueMinutes(10), null));
+        assertThat(e.getMessage(),
+                equalTo("java.lang.IllegalArgumentException: [duration] must be greater or equal to the bucket span: [10m/1h]"));
+    }
+
     private static Map<String, Object> createRecord(long timestamp, double value) {
         Map<String, Object> record = new HashMap<>();
         record.put("time", timestamp);

plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.integration;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.xpack.ml.action.DeleteJobAction;
+import org.elasticsearch.xpack.ml.action.OpenJobAction;
+import org.elasticsearch.xpack.ml.action.PutJobAction;
+import org.elasticsearch.xpack.ml.job.config.Job;
+import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
+
+/**
+ * Test that ML does not touch unnecessary indices when removing job index aliases
+ */
+public class JobStorageDeletionTaskIT extends BaseMlIntegTestCase {
+
+    private static final String UNRELATED_INDEX = "unrelated-data";
+
+    public void testUnrelatedIndexNotTouched() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(1);
+        ensureStableCluster(1);
+
+        client().admin().indices().prepareCreate(UNRELATED_INDEX).get();
+
+        enableIndexBlock(UNRELATED_INDEX, IndexMetaData.SETTING_READ_ONLY);
+
+        Job.Builder job = createJob("delete-aliases-test-job", new ByteSizeValue(2, ByteSizeUnit.MB));
+        PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
+        PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
+        assertTrue(putJobResponse.isAcknowledged());
+
+        OpenJobAction.Request openJobRequest = new OpenJobAction.Request(job.getId());
+        client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
+        awaitJobOpenedAndAssigned(job.getId(), null);
+
+        DeleteJobAction.Request deleteJobRequest = new DeleteJobAction.Request(job.getId());
+        deleteJobRequest.setForce(true);
+        client().execute(DeleteJobAction.INSTANCE, deleteJobRequest).actionGet();
+
+        // If the deletion of aliases touches the unrelated index with the block
+        // then the line above will throw a ClusterBlockException
+
+        disableIndexBlock(UNRELATED_INDEX, IndexMetaData.SETTING_READ_ONLY);
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

@@ -17,6 +17,7 @@ import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
+import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
 import org.junit.After;
 
 import java.io.BufferedReader;
@@ -648,5 +649,6 @@ public class MlJobIT extends ESRestTestCase {
     @After
     public void clearMlState() throws Exception {
         new MlRestTestStateCleaner(logger, adminClient(), this).clearMlMetadata();
+        XPackRestTestHelper.waitForPendingTasks(adminClient());
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java

@@ -233,7 +233,12 @@ abstract class MlNativeAutodetectIntegTestCase extends SecurityIntegTestCase {
     }
 
     protected void waitUntilJobIsClosed(String jobId) throws Exception {
-        assertBusy(() -> assertThat(getJobStats(jobId).get(0).getState(), equalTo(JobState.CLOSED)), 30, TimeUnit.SECONDS);
+        waitUntilJobIsClosed(jobId, TimeValue.timeValueSeconds(30));
+    }
+
+    protected void waitUntilJobIsClosed(String jobId, TimeValue waitTime) throws Exception {
+        assertBusy(() -> assertThat(getJobStats(jobId).get(0).getState(), equalTo(JobState.CLOSED)),
+                waitTime.getMillis(), TimeUnit.MILLISECONDS);
     }
 
     protected List<Job> getJob(String jobId) {