Watcher: Stop swallowing exceptions, always return them instead of message (elastic/x-pack-elasticsearch#1933)

It is really hard to debug some issues with watcher, when only the
e.getMessage() is returned as failure reasons instead of the whole
stack trace.

This commit gets rid of ExceptionsHelper.detailedMessage(e) and always
returns the whole exception.

This commit also extends the watch history to have all fields named
error be treated like an object to be sure they do not get
indexed. No matter where it's placed in the hierarchy

In addition a few Field interface classes were removed, that only contained parse fields.

relates elastic/x-pack-elasticsearch#1816

Original commit: elastic/x-pack-elasticsearch@b2ce680139

plugin/src/main/java/org/elasticsearch/xpack/common/http/HttpClient.java

@@ -154,7 +154,6 @@ public class HttpClient extends AbstractComponent {
 
         // timeouts
         if (request.connectionTimeout() != null) {
-
             config.setConnectTimeout(Math.toIntExact(request.connectionTimeout.millis()));
         } else {
             config.setConnectTimeout(Math.toIntExact(defaultConnectionTimeout.millis()));

plugin/src/main/java/org/elasticsearch/xpack/notification/hipchat/IntegrationAccount.java

@@ -90,8 +90,7 @@ public class IntegrationAccount extends HipChatAccount {
                     response));
         } catch (Exception e) {
             logger.error("failed to execute hipchat api http request", e);
-            sentMessages.add(SentMessages.SentMessage.error(room, SentMessages.SentMessage.TargetType.ROOM, message,
+            sentMessages.add(SentMessages.SentMessage.error(room, SentMessages.SentMessage.TargetType.ROOM, message, e));
-                    ExceptionsHelper.detailedMessage(e)));
         }
         return new SentMessages(name, sentMessages);
     }

plugin/src/main/java/org/elasticsearch/xpack/notification/hipchat/SentMessages.java

@@ -5,7 +5,10 @@
  */
 package org.elasticsearch.xpack.notification.hipchat;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -20,6 +23,9 @@ import java.util.Locale;
 
 public class SentMessages implements ToXContentObject, Iterable<SentMessages.SentMessage> {
 
+    private static final ParseField ACCOUNT = new ParseField("account");
+    private static final ParseField SENT_MESSAGES = new ParseField("sent_messages");
+
     private String accountName;
     private List<SentMessage> messages;
 
@@ -48,8 +54,8 @@ public class SentMessages implements ToXContentObject, Iterable<SentMessages.Sen
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(Field.ACCOUNT, accountName);
+        builder.field(ACCOUNT.getPreferredName(), accountName);
-        builder.startArray(Field.SENT_MESSAGES);
+        builder.startArray(SENT_MESSAGES.getPreferredName());
         for (SentMessage message : messages) {
             message.toXContent(builder, params);
         }
@@ -59,6 +65,11 @@ public class SentMessages implements ToXContentObject, Iterable<SentMessages.Sen
 
     public static class SentMessage implements ToXContent {
 
+        private static final ParseField STATUS = new ParseField("status");
+        private static final ParseField REQUEST = new ParseField("request");
+        private static final ParseField RESPONSE = new ParseField("response");
+        private static final ParseField MESSAGE = new ParseField("message");
+
         public enum TargetType {
             ROOM, USER;
 
@@ -70,30 +81,25 @@ public class SentMessages implements ToXContentObject, Iterable<SentMessages.Sen
         final HipChatMessage message;
         @Nullable final HttpRequest request;
         @Nullable final HttpResponse response;
-        @Nullable final String failureReason;
+        @Nullable final Exception exception;
 
         public static SentMessage responded(String targetName, TargetType targetType, HipChatMessage message, HttpRequest request,
                                             HttpResponse response) {
-            String failureReason = resolveFailureReason(response);
+            return new SentMessage(targetName, targetType, message, request, response, null);
-            return new SentMessage(targetName, targetType, message, request, response, failureReason);
         }
 
-        public static SentMessage error(String targetName, TargetType targetType, HipChatMessage message, String reason) {
+        public static SentMessage error(String targetName, TargetType targetType, HipChatMessage message, Exception e) {
-            return new SentMessage(targetName, targetType, message, null, null, reason);
+            return new SentMessage(targetName, targetType, message, null, null, e);
         }
 
         private SentMessage(String targetName, TargetType targetType, HipChatMessage message, HttpRequest request, HttpResponse response,
-                            String failureReason) {
+                            Exception exception) {
             this.targetName = targetName;
             this.targetType = targetType;
             this.message = message;
             this.request = request;
             this.response = response;
-            this.failureReason = failureReason;
+            this.exception = exception;
-        }
-
-        public boolean successful() {
-            return failureReason == null;
         }
 
         public HttpRequest getRequest() {
@@ -104,60 +110,36 @@ public class SentMessages implements ToXContentObject, Iterable<SentMessages.Sen
             return response;
         }
 
-        public String getFailureReason() {
+        public Exception getException() {
-            return failureReason;
+            return exception;
+        }
+
+        public boolean isSuccess() {
+            return response != null && response.status() >= 200 && response.status() < 300;
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            if (failureReason != null) {
+            builder.field(STATUS.getPreferredName(), isSuccess() ? "success" : "failure");
-                builder.field(Field.STATUS, "failure");
+            if (isSuccess() == false) {
-                builder.field(Field.REASON, failureReason);
+                builder.field(STATUS.getPreferredName(), "failure");
                 if (request != null) {
-                    builder.field(Field.REQUEST);
+                    builder.field(REQUEST.getPreferredName());
                     request.toXContent(builder, params);
                 }
                 if (response != null) {
-                    builder.field(Field.RESPONSE);
+                    builder.field(RESPONSE.getPreferredName());
                     response.toXContent(builder, params);
                 }
-            } else {
+                if (exception != null) {
-                builder.field(Field.STATUS, "success");
+                    ElasticsearchException.generateFailureXContent(builder, params, exception, true);
+                }
             }
             builder.field(targetType.fieldName, targetName);
-            builder.field(Field.MESSAGE);
+            builder.field(MESSAGE.getPreferredName());
             message.toXContent(builder, params, false);
             return builder.endObject();
         }
-
-        private static String resolveFailureReason(HttpResponse response) {
-            int status = response.status();
-            if (status < 300) {
-                return null;
-            }
-            switch (status) {
-                case 400:   return "Bad Request";
-                case 401:   return "Unauthorized. The provided authentication token is invalid.";
-                case 403:   return "Forbidden. The account doesn't have permission to send this message.";
-                case 404:   // Not Found
-                case 405:   // Method Not Allowed
-                case 406:   return "The account used invalid HipChat APIs"; // Not Acceptable
-                case 503:
-                case 500:   return "HipChat Server Error.";
-                default:
-                    return "Unknown Error";
-            }
-        }
-    }
-
-    interface Field {
-        String ACCOUNT = new String("account");
-        String SENT_MESSAGES = new String("sent_messages");
-        String STATUS = new String("status");
-        String REASON = new String("reason");
-        String REQUEST = new String("request");
-        String RESPONSE = new String("response");
-        String MESSAGE = new String("message");
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/notification/hipchat/UserAccount.java

@@ -88,8 +88,7 @@ public class UserAccount extends HipChatAccount {
                             response));
                 } catch (IOException e) {
                     logger.error("failed to execute hipchat api http request", e);
-                    sentMessages.add(SentMessages.SentMessage.error(room, SentMessages.SentMessage.TargetType.ROOM, message,
+                    sentMessages.add(SentMessages.SentMessage.error(room, SentMessages.SentMessage.TargetType.ROOM, message, e));
-                            ExceptionsHelper.detailedMessage(e)));
                 }
             }
         }
@@ -102,8 +101,7 @@ public class UserAccount extends HipChatAccount {
                             response));
                 } catch (Exception e) {
                     logger.error("failed to execute hipchat api http request", e);
-                    sentMessages.add(SentMessages.SentMessage.error(user, SentMessages.SentMessage.TargetType.USER, message,
+                    sentMessages.add(SentMessages.SentMessage.error(user, SentMessages.SentMessage.TargetType.USER, message, e));
-                            ExceptionsHelper.detailedMessage(e)));
                 }
             }
         }

plugin/src/main/java/org/elasticsearch/xpack/notification/hipchat/V1Account.java

@@ -84,8 +84,7 @@ public class V1Account extends HipChatAccount {
                             response));
                 } catch (Exception e) {
                     logger.error("failed to execute hipchat api http request", e);
-                    sentMessages.add(SentMessages.SentMessage.error(room, SentMessages.SentMessage.TargetType.ROOM, message,
+                    sentMessages.add(SentMessages.SentMessage.error(room, SentMessages.SentMessage.TargetType.ROOM, message, e));
-                            ExceptionsHelper.detailedMessage(e)));
                 }
             }
         }

plugin/src/main/java/org/elasticsearch/xpack/notification/slack/SentMessages.java

@@ -5,7 +5,10 @@
  */
 package org.elasticsearch.xpack.notification.slack;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -20,6 +23,9 @@ import java.util.List;
 
 public class SentMessages implements ToXContentObject, Iterable<SentMessages.SentMessage> {
 
+    private static final ParseField ACCOUNT = new ParseField("account");
+    private static final ParseField SENT_MESSAGES = new ParseField("sent_messages");
+
     private String accountName;
     private List<SentMessage> messages;
 
@@ -48,8 +54,8 @@ public class SentMessages implements ToXContentObject, Iterable<SentMessages.Sen
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(Field.ACCOUNT, accountName);
+        builder.field(ACCOUNT.getPreferredName(), accountName);
-        builder.startArray(Field.SENT_MESSAGES);
+        builder.startArray(SENT_MESSAGES.getPreferredName());
         for (SentMessage message : messages) {
             message.toXContent(builder, params);
         }
@@ -59,31 +65,32 @@ public class SentMessages implements ToXContentObject, Iterable<SentMessages.Sen
 
     public static class SentMessage implements ToXContent {
 
+        private static final ParseField STATUS = new ParseField("status");
+        private static final ParseField REQUEST = new ParseField("request");
+        private static final ParseField RESPONSE = new ParseField("response");
+        private static final ParseField TO = new ParseField("to");
+        private static final ParseField MESSAGE = new ParseField("message");
+
         final String to;
         final SlackMessage message;
         @Nullable final HttpRequest request;
         @Nullable final HttpResponse response;
-        @Nullable final String failureReason;
+        @Nullable final Exception exception;
 
         public static SentMessage responded(String to, SlackMessage message, HttpRequest request, HttpResponse response) {
-            String failureReason = resolveFailureReason(response);
+            return new SentMessage(to, message, request, response, null);
-            return new SentMessage(to, message, request, response, failureReason);
         }
 
-        public static SentMessage error(String to, SlackMessage message, String reason) {
+        public static SentMessage error(String to, SlackMessage message, Exception e) {
-            return new SentMessage(to, message, null, null, reason);
+            return new SentMessage(to, message, null, null, e);
         }
 
-        private SentMessage(String to, SlackMessage message, HttpRequest request, HttpResponse response, String failureReason) {
+        private SentMessage(String to, SlackMessage message, HttpRequest request, HttpResponse response, Exception exception) {
             this.to = to;
             this.message = message;
             this.request = request;
             this.response = response;
-            this.failureReason = failureReason;
+            this.exception = exception;
-        }
-
-        public boolean successful() {
-            return failureReason == null;
         }
 
         public HttpRequest getRequest() {
@@ -94,54 +101,37 @@ public class SentMessages implements ToXContentObject, Iterable<SentMessages.Sen
             return response;
         }
 
+        public Exception getException() {
+            return exception;
+        }
+
+        public boolean isSuccess() {
+            return response != null && response.status() >= 200 && response.status() < 300;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            if (failureReason != null) {
+            builder.field(STATUS.getPreferredName(), isSuccess() ? "success" : "failure");
-                builder.field(Field.STATUS, "failure");
+            if (isSuccess() == false) {
-                builder.field(Field.REASON, failureReason);
                 if (request != null) {
-                    builder.field(Field.REQUEST);
+                    builder.field(REQUEST.getPreferredName());
                     request.toXContent(builder, params);
                 }
                 if (response != null) {
-                    builder.field(Field.RESPONSE);
+                    builder.field(RESPONSE.getPreferredName());
                     response.toXContent(builder, params);
                 }
-            } else {
+                if (exception != null) {
-                builder.field(Field.STATUS, "success");
+                    ElasticsearchException.generateFailureXContent(builder, params, exception, true);
+                }
             }
             if (to != null) {
-                builder.field(Field.TO, to);
+                builder.field(TO.getPreferredName(), to);
             }
-            builder.field(Field.MESSAGE);
+            builder.field(MESSAGE.getPreferredName());
             message.toXContent(builder, params, false);
             return builder.endObject();
         }
-
-        private static String resolveFailureReason(HttpResponse response) {
-            int status = response.status();
-            if (status < 300) {
-                return null;
-            }
-            if (status > 399 && status < 500) {
-                return "Bad Request";
-            }
-            if (status > 499) {
-                return "Slack Server Error";
-            }
-            return "Unknown Error";
-        }
-    }
-
-    interface Field {
-        String ACCOUNT = new String("account");
-        String SENT_MESSAGES = new String("sent_messages");
-        String STATUS = new String("status");
-        String REASON = new String("reason");
-        String REQUEST = new String("request");
-        String RESPONSE = new String("response");
-        String MESSAGE = new String("message");
-        String TO = new String("to");
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/notification/slack/SlackAccount.java

@@ -111,7 +111,7 @@ public class SlackAccount {
             return SentMessages.SentMessage.responded(to, message, request, response);
         } catch (Exception e) {
             logger.error("failed to execute slack api http request", e);
-            return SentMessages.SentMessage.error(to, message, ExceptionsHelper.detailedMessage(e));
+            return SentMessages.SentMessage.error(to, message, e);
         }
     }
 

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/Action.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.actions;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -58,6 +59,8 @@ public interface Action extends ToXContentObject {
          */
         public static class StoppedResult extends Result {
 
+            private static ParseField REASON = new ParseField("reason");
+
             private final String reason;
 
             protected StoppedResult(String type, Status status, String reason, Object... args) {
@@ -71,7 +74,7 @@ public interface Action extends ToXContentObject {
 
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                return builder.field(Field.REASON.getPreferredName(), reason);
+                return builder.field(REASON.getPreferredName(), reason);
             }
 
         }
@@ -85,7 +88,26 @@ public interface Action extends ToXContentObject {
             public Failure(String type, String reason, Object... args) {
                 super(type, Status.FAILURE, reason, args);
             }
+        }
 
+        public static class FailureWithException extends Result {
+
+            private final Exception exception;
+
+            public FailureWithException(String type, Exception exception) {
+                super(type, Status.FAILURE);
+                this.exception = exception;
+            }
+
+            public Exception getException() {
+                return exception;
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                ElasticsearchException.generateFailureXContent(builder, params, exception, true);
+                return builder;
+            }
         }
 
         /**
@@ -127,8 +149,4 @@ public interface Action extends ToXContentObject {
 
         A build();
     }
-
-    interface Field {
-        ParseField REASON = new ParseField("reason");
-    }
 }

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/ActionWrapper.java

@@ -8,7 +8,6 @@ package org.elasticsearch.xpack.watcher.actions;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.unit.TimeValue;
@@ -138,9 +137,7 @@ public class ActionWrapper implements ToXContentObject {
                 action.logger().error(
                         (Supplier<?>) () -> new ParameterizedMessage(
                                 "failed to execute action [{}/{}]. failed to transform payload.", ctx.watch().id(), id), e);
-                return new ActionWrapper.Result(id, conditionResult, null,
+                return new ActionWrapper.Result(id, conditionResult, null, new Action.Result.FailureWithException(action.type(), e));
-                                                new Action.Result.Failure(action.type(), "Failed to transform payload. error: {}",
-                                                    ExceptionsHelper.detailedMessage(e)));
             }
         }
         try {
@@ -149,7 +146,7 @@ public class ActionWrapper implements ToXContentObject {
         } catch (Exception e) {
             action.logger().error(
                     (Supplier<?>) () -> new ParameterizedMessage("failed to execute action [{}/{}]", ctx.watch().id(), id), e);
-            return new ActionWrapper.Result(id, new Action.Result.Failure(action.type(), ExceptionsHelper.detailedMessage(e)));
+            return new ActionWrapper.Result(id, new Action.Result.FailureWithException(action.type(), e));
         }
     }
 
@@ -160,19 +157,15 @@ public class ActionWrapper implements ToXContentObject {
 
         ActionWrapper that = (ActionWrapper) o;
 
-        if (!id.equals(that.id)) return false;
+        return Objects.equals(id, that.id) &&
-        if (condition != null ? !condition.equals(that.condition) : that.condition != null) return false;
+                Objects.equals(condition, that.condition) &&
-        if (transform != null ? !transform.equals(that.transform) : that.transform != null) return false;
+                Objects.equals(transform, that.transform) &&
-        return action.equals(that.action);
+                Objects.equals(action, that.action);
     }
 
     @Override
     public int hashCode() {
-        int result = id.hashCode();
+        return Objects.hash(id, condition, transform, action);
-        result = 31 * result + (condition != null ? condition.hashCode() : 0);
-        result = 31 * result + (transform != null ? transform.hashCode() : 0);
-        result = 31 * result + action.hashCode();
-        return result;
     }
 
     @Override
@@ -189,7 +182,7 @@ public class ActionWrapper implements ToXContentObject {
                     .endObject();
         }
         if (transform != null) {
-            builder.startObject(Transform.Field.TRANSFORM.getPreferredName())
+            builder.startObject(Transform.TRANSFORM.getPreferredName())
                     .field(transform.type(), transform, params)
                     .endObject();
         }
@@ -215,7 +208,7 @@ public class ActionWrapper implements ToXContentObject {
             } else {
                 if (Watch.Field.CONDITION.match(currentFieldName)) {
                     condition = actionRegistry.getConditionRegistry().parseExecutable(watchId, parser);
-                } else if (Transform.Field.TRANSFORM.match(currentFieldName)) {
+                } else if (Transform.TRANSFORM.match(currentFieldName)) {
                     transform = actionRegistry.getTransformRegistry().parse(watchId, parser);
                 } else if (Throttler.Field.THROTTLE_PERIOD.match(currentFieldName)) {
                     throttlePeriod = timeValueMillis(parser.longValue());
@@ -309,7 +302,7 @@ public class ActionWrapper implements ToXContentObject {
                 builder.field(Watch.Field.CONDITION.getPreferredName(), condition, params);
             }
             if (transform != null) {
-                builder.field(Transform.Field.TRANSFORM.getPreferredName(), transform, params);
+                builder.field(Transform.TRANSFORM.getPreferredName(), transform, params);
             }
             action.toXContent(builder, params);
             return builder.endObject();

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/email/EmailAction.java

@@ -282,7 +282,7 @@ public class EmailAction implements Action {
         }
     }
 
-    interface Field extends Action.Field {
+    interface Field {
 
         // common fields
         ParseField ACCOUNT = new ParseField("account");

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/email/ExecutableEmailAction.java

@@ -57,7 +57,7 @@ public class ExecutableEmailAction extends ExecutableAction<EmailAction> {
                     Attachment attachment = parser.toAttachment(ctx, payload, emailAttachment);
                     attachments.put(attachment.id(), attachment);
                 } catch (ElasticsearchException | IOException e) {
-                    return new EmailAction.Result.Failure(action.type(), e.getMessage());
+                    return new EmailAction.Result.FailureWithException(action.type(), e);
                 }
             }
         }

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/hipchat/HipChatAction.java

@@ -137,7 +137,7 @@ public class HipChatAction implements Action {
                 boolean hasSuccesses = false;
                 boolean hasFailures = false;
                 for (SentMessages.SentMessage message : sentMessages) {
-                    if (message.successful()) {
+                    if (message.isSuccess()) {
                         hasSuccesses = true;
                     } else {
                         hasFailures = true;

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/index/IndexAction.java

@@ -287,7 +287,7 @@ public class IndexAction implements Action {
         }
     }
 
-    interface Field extends Action.Field {
+    interface Field {
         ParseField INDEX = new ParseField("index");
         ParseField DOC_TYPE = new ParseField("doc_type");
         ParseField DOC_ID = new ParseField("doc_id");

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/LoggingAction.java

@@ -186,7 +186,7 @@ public class LoggingAction implements Action {
         }
     }
 
-    interface Field extends Action.Field {
+    interface Field {
         ParseField CATEGORY = new ParseField("category");
         ParseField LEVEL = new ParseField("level");
         ParseField TEXT = new ParseField("text");

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/slack/SlackAction.java

@@ -136,7 +136,7 @@ public class SlackAction implements Action {
                 boolean hasSuccesses = false;
                 boolean hasFailures = false;
                 for (SentMessages.SentMessage message : sentMessages) {
-                    if (message.successful()) {
+                    if (message.isSuccess()) {
                         hasSuccesses = true;
                     } else {
                         hasFailures = true;

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/ExecutableWebhookAction.java

@@ -41,11 +41,10 @@ public class ExecutableWebhookAction extends ExecutableAction<WebhookAction> {
 
         HttpResponse response = httpClient.execute(request);
 
-        int status = response.status();
+        if (response.status() >= 400) {
-        if (status >= 400) {
-            logger.warn("received http status [{}] when connecting to watch action [{}/{}/{}]", status, ctx.watch().id(), type(), actionId);
             return new WebhookAction.Result.Failure(request, response);
+        } else {
+            return new WebhookAction.Result.Success(request, response);
         }
-        return new WebhookAction.Result.Success(request, response);
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookAction.java

@@ -9,10 +9,10 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.watcher.actions.Action;
 import org.elasticsearch.xpack.common.http.HttpRequest;
 import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.common.http.HttpResponse;
+import org.elasticsearch.xpack.watcher.actions.Action;
 
 import java.io.IOException;
 
@@ -170,7 +170,7 @@ public class WebhookAction implements Action {
         }
     }
 
-    interface Field extends Action.Field {
+    interface Field {
         ParseField REQUEST = new ParseField("request");
         ParseField RESPONSE = new ParseField("response");
     }

plugin/src/main/java/org/elasticsearch/xpack/watcher/client/WatchSourceBuilder.java

@@ -195,7 +195,7 @@ public class WatchSourceBuilder extends ToXContentToBytes implements ToXContent
                         .endObject();
             }
             if (transform != null) {
-                builder.startObject(Transform.Field.TRANSFORM.getPreferredName())
+                builder.startObject(Transform.TRANSFORM.getPreferredName())
                         .field(transform.type(), transform, params)
                         .endObject();
             }

plugin/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java

@@ -200,8 +200,7 @@ public class ExecutionService extends AbstractComponent {
                 e -> {
                     Throwable cause = ExceptionsHelper.unwrapCause(e);
                     if (cause instanceof EsRejectedExecutionException) {
-                        logger.debug("failed to store watch records due to overloaded threadpool [{}]",
+                        logger.debug("failed to store watch records due to filled up watcher threadpool");
-                                ExceptionsHelper.detailedMessage(e));
                     } else {
                         logger.warn("failed to store watch records", e);
                     }

plugin/src/main/java/org/elasticsearch/xpack/watcher/execution/WatchExecutionResult.java

@@ -82,7 +82,7 @@ public class WatchExecutionResult implements ToXContentObject {
             builder.field(Field.CONDITION.getPreferredName(), conditionResult, params);
         }
         if (transformResult != null) {
-            builder.field(Transform.Field.TRANSFORM.getPreferredName(), transformResult, params);
+            builder.field(Transform.TRANSFORM.getPreferredName(), transformResult, params);
         }
         builder.startArray(Field.ACTIONS.getPreferredName());
         for (ActionWrapper.Result result : actionsResults.values()) {

plugin/src/main/java/org/elasticsearch/xpack/watcher/input/Input.java

@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.watcher.input;
 
-import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -20,27 +21,31 @@ public interface Input extends ToXContentObject {
 
     abstract class Result implements ToXContentObject {
 
+        private static final ParseField STATUS = new ParseField("status");
+        private static final ParseField TYPE = new ParseField("type");
+        private static final ParseField PAYLOAD = new ParseField("payload");
+
         public enum Status {
             SUCCESS, FAILURE
         }
 
         protected final String type;
         protected final Status status;
-        private final String reason;
         private final Payload payload;
+        @Nullable private final Exception exception;
 
         protected Result(String type, Payload payload) {
             this.status = Status.SUCCESS;
             this.type = type;
             this.payload = payload;
-            this.reason = null;
+            this.exception = null;
         }
 
         protected Result(String type, Exception e) {
             this.status = Status.FAILURE;
             this.type = type;
-            this.reason = ExceptionsHelper.detailedMessage(e);
             this.payload = Payload.EMPTY;
+            this.exception = e;
         }
 
         public String type() {
@@ -55,24 +60,24 @@ public interface Input extends ToXContentObject {
             return payload;
         }
 
-        public String reason() {
+        public Exception getException() {
             assert status == Status.FAILURE;
-            return reason;
+            return exception;
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(Field.TYPE.getPreferredName(), type);
+            builder.field(TYPE.getPreferredName(), type);
-            builder.field(Field.STATUS.getPreferredName(), status.name().toLowerCase(Locale.ROOT));
+            builder.field(STATUS.getPreferredName(), status.name().toLowerCase(Locale.ROOT));
             switch (status) {
                 case SUCCESS:
                     assert payload != null;
-                    builder.field(Field.PAYLOAD.getPreferredName(), payload, params);
+                    builder.field(PAYLOAD.getPreferredName(), payload, params);
                     break;
                 case FAILURE:
-                    assert reason != null;
+                    assert exception != null;
-                    builder.field(Field.REASON.getPreferredName(), reason);
+                    ElasticsearchException.generateFailureXContent(builder, params, exception, true);
                     break;
                 default:
                     assert false;
@@ -87,11 +92,4 @@ public interface Input extends ToXContentObject {
     interface Builder<I extends Input> {
         I build();
     }
-
-    interface Field {
-        ParseField STATUS = new ParseField("status");
-        ParseField TYPE = new ParseField("type");
-        ParseField PAYLOAD = new ParseField("payload");
-        ParseField REASON = new ParseField("reason");
-    }
 }

plugin/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java

@@ -202,7 +202,7 @@ public class HttpInput implements Input {
         }
     }
 
-    interface Field extends Input.Field {
+    interface Field {
         ParseField REQUEST = new ParseField("request");
         ParseField EXTRACT = new ParseField("extract");
         ParseField STATUS_CODE = new ParseField("status_code");

plugin/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInput.java

@@ -233,7 +233,7 @@ public class SearchInput implements Input {
         }
     }
 
-    public interface Field extends Input.Field {
+    public interface Field {
         ParseField REQUEST = new ParseField("request");
         ParseField EXTRACT = new ParseField("extract");
         ParseField TIMEOUT = new ParseField("timeout_in_millis");

plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

@@ -36,8 +36,9 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
     // version 2: added mappings for jira action
     // version 3: include watch status in history
     // version 6: upgrade to ES 6, removal of _status field
+    // version 7: add full exception stack traces for better debugging
     // Note: if you change this, also inform the kibana team around the watcher-ui
-    public static final String INDEX_TEMPLATE_VERSION = "6";
+    public static final String INDEX_TEMPLATE_VERSION = "7";
 
     public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";

plugin/src/main/java/org/elasticsearch/xpack/watcher/transform/Transform.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.transform;
 
-import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -18,10 +18,17 @@ import java.util.Locale;
 
 public interface Transform extends ToXContent {
 
+    ParseField TRANSFORM = new ParseField("transform");
+
     String type();
 
     abstract class Result implements ToXContentObject {
 
+        private static final ParseField TYPE = new ParseField("type");
+        private static final ParseField STATUS = new ParseField("status");
+        private static final ParseField PAYLOAD = new ParseField("payload");
+        private static final ParseField REASON = new ParseField("reason");
+
         public enum Status {
             SUCCESS, FAILURE
         }
@@ -30,23 +37,30 @@ public interface Transform extends ToXContent {
         protected final Status status;
         @Nullable protected final Payload payload;
         @Nullable protected final String reason;
+        @Nullable protected final Exception exception;
 
         public Result(String type, Payload payload) {
             this.type = type;
             this.status = Status.SUCCESS;
             this.payload = payload;
             this.reason = null;
+            this.exception = null;
+        }
+
+        public Result(String type, String reason) {
+            this.type = type;
+            this.status = Status.FAILURE;
+            this.reason = reason;
+            this.payload = null;
+            this.exception = null;
         }
 
         public Result(String type, Exception e) {
-            this(type, ExceptionsHelper.detailedMessage(e));
-        }
-
-        public Result(String type, String errorMessage) {
             this.type = type;
             this.status = Status.FAILURE;
-            this.reason = errorMessage;
+            this.reason = e.getMessage();
             this.payload = null;
+            this.exception = e;
         }
 
         public String type() {
@@ -70,16 +84,17 @@ public interface Transform extends ToXContent {
         @Override
         public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(Field.TYPE.getPreferredName(), type);
+            builder.field(TYPE.getPreferredName(), type);
-            builder.field(Field.STATUS.getPreferredName(), status.name().toLowerCase(Locale.ROOT));
+            builder.field(STATUS.getPreferredName(), status.name().toLowerCase(Locale.ROOT));
             switch (status) {
                 case SUCCESS:
-                    assert reason == null;
+                    assert exception == null;
-                    builder.field(Field.PAYLOAD.getPreferredName(), payload, params);
+                    builder.field(PAYLOAD.getPreferredName(), payload, params);
                     break;
                 case FAILURE:
                     assert payload == null;
-                    builder.field(Field.REASON.getPreferredName(), reason);
+                    builder.field(REASON.getPreferredName(), reason);
+                    ElasticsearchException.generateFailureXContent(builder, params, exception, true);
                     break;
                 default:
                     assert false;
@@ -96,14 +111,4 @@ public interface Transform extends ToXContent {
 
         T build();
     }
-
-    interface Field {
-        ParseField TRANSFORM = new ParseField("transform");
-
-        ParseField TYPE = new ParseField("type");
-        ParseField STATUS = new ParseField("status");
-        ParseField PAYLOAD = new ParseField("payload");
-        ParseField REASON = new ParseField("reason");
-
-    }
 }

plugin/src/main/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransform.java

@@ -162,7 +162,7 @@ public class ChainTransform implements Transform {
         }
     }
 
-    interface Field extends Transform.Field {
+    interface Field {
         ParseField RESULTS = new ParseField("results");
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransform.java

@@ -189,7 +189,7 @@ public class SearchTransform implements Transform {
         }
     }
 
-    public interface Field extends Transform.Field {
+    public interface Field {
         ParseField REQUEST = new ParseField("request");
         ParseField TIMEOUT = new ParseField("timeout_in_millis");
         ParseField TIMEOUT_HUMAN = new ParseField("timeout");

plugin/src/main/resources/watch-history.json

@@ -30,6 +30,16 @@
             }
           }
         },
+        {
+          "disabled_exception_fields": {
+            "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*)|actions)\\.error",
+            "match_pattern": "regex",
+            "mapping": {
+              "type": "object",
+              "enabled": false
+            }
+          }
+        },
         {
           "disabled_jira_custom_fields": {
             "path_match":   "result.actions.jira.fields.customfield_*",

plugin/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionTests.java

@@ -553,9 +553,9 @@ public class EmailActionTests extends ESTestCase {
                 .buildMock();
 
         Action.Result result = executableEmailAction.execute("test", ctx, new Payload.Simple());
-        assertThat(result, instanceOf(EmailAction.Result.Failure.class));
+        assertThat(result, instanceOf(EmailAction.Result.FailureWithException.class));
-        EmailAction.Result.Failure failure = (EmailAction.Result.Failure) result;
+        EmailAction.Result.FailureWithException failure = (EmailAction.Result.FailureWithException) result;
-        assertThat(failure.reason(),
+        assertThat(failure.getException().getMessage(),
                 is("Watch[watch1] attachment[second] HTTP error status host[localhost], port[80], method[GET], path[/second], " +
                         "status[403]"));
     }

plugin/src/test/java/org/elasticsearch/xpack/watcher/actions/slack/SlackActionTests.java

@@ -110,9 +110,10 @@ public class SlackActionTests extends ESTestCase {
         for (int i = 0; i < count; i++) {
             HttpResponse response = mock(HttpResponse.class);
             HttpRequest request = mock(HttpRequest.class);
-            switch (randomIntBetween(0, 2)) {
+            int randomInt = randomIntBetween(0, 2);
+            switch (randomInt) {
                 case 0:
-                    messages.add(SentMessages.SentMessage.error(randomAlphaOfLength(10), message, "unknown error"));
+                    messages.add(SentMessages.SentMessage.error(randomAlphaOfLength(10), message, new Exception("unknown error")));
                     hasError = true;
                     break;
                 case 1:

plugin/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java

@@ -45,6 +45,7 @@ import org.elasticsearch.xpack.watcher.watch.WatchStatus;
 import org.joda.time.DateTime;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -229,7 +230,7 @@ public class ExecutionServiceTests extends ESTestCase {
         input = mock(ExecutableInput.class);
         Input.Result inputResult = mock(Input.Result.class);
         when(inputResult.status()).thenReturn(Input.Result.Status.FAILURE);
-        when(inputResult.reason()).thenReturn("_reason");
+        when(inputResult.getException()).thenReturn(new IOException());
         when(input.execute(eq(context), any(Payload.class))).thenReturn(inputResult);
 
         Condition.Result conditionResult = AlwaysCondition.RESULT_INSTANCE;

plugin/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java

@@ -5,7 +5,12 @@
  */
 package org.elasticsearch.xpack.watcher.history;
 
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.test.http.MockResponse;
@@ -15,20 +20,31 @@ import org.elasticsearch.xpack.common.http.HttpMethod;
 import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.condition.AlwaysCondition;
 import org.elasticsearch.xpack.watcher.execution.ExecutionState;
+import org.elasticsearch.xpack.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchResponse;
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.webhookAction;
 import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
 import static org.elasticsearch.xpack.watcher.input.InputBuilders.httpInput;
 import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
 import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
 /**
@@ -111,4 +127,64 @@ public class HistoryTemplateHttpMappingsTests extends AbstractWatcherIntegration
         assertThat(webServer.requests().get(0).getUri().getPath(), is("/input/path"));
         assertThat(webServer.requests().get(1).getUri().getPath(), is("/webhook/path"));
     }
+
+    public void testExceptionMapping() {
+        // delete all history indices to ensure that we start with a fresh mapping
+        assertAcked(client().admin().indices().prepareDelete(HistoryStore.INDEX_PREFIX + "*"));
+
+        String id = randomAlphaOfLength(10);
+        // switch between delaying the input or the action http request
+        boolean abortAtInput = randomBoolean();
+        if (abortAtInput) {
+            webServer.enqueue(new MockResponse().setBeforeReplyDelay(TimeValue.timeValueSeconds(5)));
+        } else {
+            webServer.enqueue(new MockResponse().setBody("{}"));
+            webServer.enqueue(new MockResponse().setBeforeReplyDelay(TimeValue.timeValueSeconds(5)));
+        }
+
+        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch(id).setSource(watchBuilder()
+                .trigger(schedule(interval("5s")))
+                .input(httpInput(HttpRequestTemplate.builder("localhost", webServer.getPort())
+                        .path("/")
+                        .readTimeout(TimeValue.timeValueMillis(10))))
+                .condition(AlwaysCondition.INSTANCE)
+                .addAction("_webhook", webhookAction(HttpRequestTemplate.builder("localhost", webServer.getPort())
+                        .readTimeout(TimeValue.timeValueMillis(10))
+                        .path("/webhook/path")
+                        .method(HttpMethod.POST)
+                        .body("_body"))))
+                .get();
+
+        assertThat(putWatchResponse.isCreated(), is(true));
+        watcherClient().prepareExecuteWatch(id).setRecordExecution(true).get();
+
+        // ensure watcher history index has been written with this id
+        flushAndRefresh(HistoryStore.INDEX_PREFIX + "*");
+        SearchResponse searchResponse = client().prepareSearch(HistoryStore.INDEX_PREFIX + "*")
+                .setQuery(QueryBuilders.termQuery("watch_id", id))
+                .get();
+        assertHitCount(searchResponse, 1L);
+
+        // ensure that enabled is set to false
+        List<Boolean> indexed = new ArrayList<>();
+        GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings(HistoryStore.INDEX_PREFIX + "*").get();
+        Iterator<ImmutableOpenMap<String, MappingMetaData>> iterator = mappingsResponse.getMappings().valuesIt();
+        while (iterator.hasNext()) {
+            ImmutableOpenMap<String, MappingMetaData> mapping = iterator.next();
+            assertThat(mapping.containsKey("doc"), is(true));
+            Map<String, Object> docMapping = mapping.get("doc").getSourceAsMap();
+            if (abortAtInput) {
+                Boolean enabled = ObjectPath.eval("properties.result.properties.input.properties.error.enabled", docMapping);
+                indexed.add(enabled);
+            } else {
+                Boolean enabled = ObjectPath.eval("properties.result.properties.actions.properties.error.enabled", docMapping);
+                indexed.add(enabled);
+            }
+        }
+
+        assertThat(indexed, hasSize(greaterThanOrEqualTo(1)));
+        logger.info("GOT [{}]", indexed);
+        assertThat(indexed, hasItem(false));
+        assertThat(indexed, not(hasItem(true)));
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputTests.java

@@ -55,6 +55,7 @@ import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 import static org.joda.time.DateTimeZone.UTC;
 
 public class ChainInputTests extends ESTestCase {
@@ -165,7 +166,8 @@ public class ChainInputTests extends ESTestCase {
 
         XContentBuilder builder = jsonBuilder();
         chainedResult.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertThat(builder.bytes().utf8ToString(), containsString("\"reason\":\"ElasticsearchException[foo]\""));
+        assertThat(builder.bytes().utf8ToString(), containsString("\"type\":\"exception\""));
+        assertThat(builder.bytes().utf8ToString(), containsString("\"reason\":\"foo\""));
     }
 
     /* https://github.com/elastic/x-plugins/issues/3736

plugin/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java

@@ -10,7 +10,10 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
@@ -33,6 +36,7 @@ import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.watcher.input.InputBuilders;
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
+import org.elasticsearch.xpack.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent;
@@ -42,6 +46,7 @@ import org.elasticsearch.xpack.watcher.watch.WatchStatus;
 import org.joda.time.DateTime;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -60,7 +65,9 @@ import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
 import static org.joda.time.DateTimeZone.UTC;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
@@ -325,6 +332,32 @@ public class HttpInputTests extends ESTestCase {
         assertThat(data.get(1).get("foo"), is("second"));
     }
 
+    public void testExceptionCase() throws Exception {
+        when(httpClient.execute(any(HttpRequest.class))).thenThrow(new IOException("could not connect"));
+
+        HttpRequestTemplate.Builder request = HttpRequestTemplate.builder("localhost", 8080);
+        HttpInput httpInput = InputBuilders.httpInput(request.build()).build();
+        ExecutableHttpInput input = new ExecutableHttpInput(httpInput, logger, httpClient, templateEngine);
+
+        WatchExecutionContext ctx = createWatchExecutionContext();
+        HttpInput.Result result = input.execute(ctx, new Payload.Simple());
+
+        assertThat(result.getException(), is(notNullValue()));
+        assertThat(result.getException(), is(instanceOf(IOException.class)));
+        assertThat(result.getException().getMessage(), is("could not connect"));
+
+        try (XContentBuilder builder = jsonBuilder()) {
+            result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            BytesReference bytes = builder.bytes();
+            try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, bytes)) {
+                Map<String, Object> data = parser.map();
+                String reason = ObjectPath.eval("error.reason", data);
+                assertThat(reason, is("could not connect"));
+                String type = ObjectPath.eval("error.type", data);
+                assertThat(type, is("i_o_exception"));
+            }
+        }
+    }
 
     private WatchExecutionContext createWatchExecutionContext() {
         Watch watch = new Watch("test-watch",

plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HipChatServiceTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.test.integration;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.junit.annotations.Network;
@@ -201,7 +202,10 @@ public class HipChatServiceTests extends AbstractWatcherIntegrationTestCase {
         for (SentMessages.SentMessage message : messages) {
             logger.info("Request: [{}]", message.getRequest());
             logger.info("Response: [{}]", message.getResponse());
-            assertThat("Expected no failures, but got [" + message.getFailureReason() + "]", message.successful(), is(true));
+            if (message.getException() != null) {
+                logger.info("Exception stacktrace: [{}]", ExceptionsHelper.stackTrace(message.getException()));
+            }
+            assertThat(message.isSuccess(), is(true));
             assertThat(message.getRequest(), notNullValue());
             assertThat(message.getResponse(), notNullValue());
             assertThat(message.getResponse().status(), lessThan(300));

plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SlackServiceTests.java

@@ -38,6 +38,7 @@ import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interva
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 @Network
 public class SlackServiceTests extends AbstractWatcherIntegrationTestCase {
@@ -78,7 +79,7 @@ public class SlackServiceTests extends AbstractWatcherIntegrationTestCase {
         assertThat(messages.count(), is(2));
         for (SentMessages.SentMessage sentMessage : messages) {
             try {
-                assertThat(sentMessage.successful(), is(true));
+                assertThat(sentMessage.getException(), is(nullValue()));
                 assertThat(sentMessage.getRequest(), notNullValue());
                 assertThat(sentMessage.getResponse(), notNullValue());
                 assertThat(sentMessage.getResponse().status(), lessThan(300));

plugin/src/test/resources/rest-api-spec/test/watcher/execute_watch/20_transform.yml

@@ -192,4 +192,5 @@ setup:
   - match: { watch_record.trigger_event.type: "manual" }
   - match: { watch_record.state: "executed" }
   - match: { watch_record.result.transform.status: "failure" }
-  - match: { watch_record.result.transform.reason: "ParsingException[no [query] registered for [does_not_exist]]" }
+  - match: { watch_record.result.transform.reason: "no [query] registered for [does_not_exist]" }
+  - is_true: watch_record.result.transform.error