[7.x][Transform] decouple task and indexer (#48812)

decouple TransformTask and ClientTransformIndexer. Interaction between the 2 classes are
now moved into a context class which holds shared information.

relates #45369

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java

@@ -258,6 +258,10 @@ public class TransformCheckpoint implements Writeable, ToXContentObject {
         return NAME + "-" + transformId + "-" + checkpoint;
     }
 
+    public static boolean isNullOrEmpty (TransformCheckpoint checkpoint) {
+        return checkpoint == null || checkpoint.isEmpty();
+    }
+
     /**
      * Calculate the diff of 2 checkpoints
      *

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java

@@ -98,7 +98,6 @@ import org.elasticsearch.xpack.transform.rest.action.compat.RestStartTransformAc
 import org.elasticsearch.xpack.transform.rest.action.compat.RestStopTransformActionDeprecated;
 import org.elasticsearch.xpack.transform.rest.action.compat.RestUpdateTransformActionDeprecated;
 import org.elasticsearch.xpack.transform.transforms.TransformPersistentTasksExecutor;
-import org.elasticsearch.xpack.transform.transforms.TransformTask;
 
 import java.io.IOException;
 import java.time.Clock;
@@ -128,6 +127,19 @@ public class Transform extends Plugin implements ActionPlugin, PersistentTaskPlu
     private final SetOnce<TransformCheckpointService> transformCheckpointService = new SetOnce<>();
     private final SetOnce<SchedulerEngine> schedulerEngine = new SetOnce<>();
 
+    public static final int DEFAULT_FAILURE_RETRIES = 10;
+
+    // How many times the transform task can retry on an non-critical failure
+    public static final Setting<Integer> NUM_FAILURE_RETRIES_SETTING = Setting
+        .intSetting(
+            "xpack.transform.num_transform_failure_retries",
+            DEFAULT_FAILURE_RETRIES,
+            0,
+            100,
+            Setting.Property.NodeScope,
+            Setting.Property.Dynamic
+        );
+
     public Transform(Settings settings) {
         this.settings = settings;
         this.enabled = XPackSettings.TRANSFORM_ENABLED.get(settings);
@@ -146,18 +158,27 @@ public class Transform extends Plugin implements ActionPlugin, PersistentTaskPlu
         return modules;
     }
 
-    protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
+    }
 
     @Override
-    public List<RestHandler> getRestHandlers(final Settings settings, final RestController restController,
+    public List<RestHandler> getRestHandlers(
-            final ClusterSettings clusterSettings, final IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter,
+        final Settings settings,
-            final IndexNameExpressionResolver indexNameExpressionResolver, final Supplier<DiscoveryNodes> nodesInCluster) {
+        final RestController restController,
+        final ClusterSettings clusterSettings,
+        final IndexScopedSettings indexScopedSettings,
+        final SettingsFilter settingsFilter,
+        final IndexNameExpressionResolver indexNameExpressionResolver,
+        final Supplier<DiscoveryNodes> nodesInCluster
+    ) {
 
         if (!enabled) {
             return emptyList();
         }
 
-        return Arrays.asList(
+        return Arrays
+            .asList(
                 new RestPutTransformAction(restController),
                 new RestStartTransformAction(restController),
                 new RestStopTransformAction(restController),
@@ -185,7 +206,8 @@ public class Transform extends Plugin implements ActionPlugin, PersistentTaskPlu
             return emptyList();
         }
 
-        return Arrays.asList(
+        return Arrays
+            .asList(
                 new ActionHandler<>(PutTransformAction.INSTANCE, TransportPutTransformAction.class),
                 new ActionHandler<>(StartTransformAction.INSTANCE, TransportStartTransformAction.class),
                 new ActionHandler<>(StopTransformAction.INSTANCE, TransportStopTransformAction.class),
@@ -213,35 +235,45 @@ public class Transform extends Plugin implements ActionPlugin, PersistentTaskPlu
             return emptyList();
         }
 
-        FixedExecutorBuilder indexing = new FixedExecutorBuilder(settings, TASK_THREAD_POOL_NAME, 4, 4,
+        FixedExecutorBuilder indexing = new FixedExecutorBuilder(settings, TASK_THREAD_POOL_NAME, 4, 4, "transform.task_thread_pool");
-                "transform.task_thread_pool");
 
         return Collections.singletonList(indexing);
     }
 
     @Override
-    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+    public Collection<Object> createComponents(
-            ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
+        Client client,
-            Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry
+    ) {
         if (enabled == false || transportClientMode) {
             return emptyList();
         }
         transformAuditor.set(new TransformAuditor(client, clusterService.getNodeName()));
         transformConfigManager.set(new TransformConfigManager(client, xContentRegistry));
-        transformCheckpointService.set(new TransformCheckpointService(client,
+        transformCheckpointService.set(new TransformCheckpointService(client, transformConfigManager.get(), transformAuditor.get()));
-                                                                      transformConfigManager.get(),
-                                                                      transformAuditor.get()));
 
-        return Arrays.asList(transformConfigManager.get(), transformAuditor.get(), transformCheckpointService.get(),
+        return Arrays
-                new TransformClusterStateListener(clusterService, client));
+            .asList(
+                transformConfigManager.get(),
+                transformAuditor.get(),
+                transformCheckpointService.get(),
+                new TransformClusterStateListener(clusterService, client)
+            );
     }
 
     @Override
     public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
         return templates -> {
             try {
-                templates.put(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME,
+                templates
-                        TransformInternalIndex.getIndexTemplateMetaData());
+                    .put(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME, TransformInternalIndex.getIndexTemplateMetaData());
             } catch (IOException e) {
                 logger.error("Error creating data frame index template", e);
             }
@@ -255,8 +287,12 @@ public class Transform extends Plugin implements ActionPlugin, PersistentTaskPlu
     }
 
     @Override
-    public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterService clusterService, ThreadPool threadPool,
+    public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(
-            Client client, SettingsModule settingsModule) {
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        Client client,
+        SettingsModule settingsModule
+    ) {
         if (enabled == false || transportClientMode) {
             return emptyList();
         }
@@ -269,20 +305,24 @@ public class Transform extends Plugin implements ActionPlugin, PersistentTaskPlu
         assert transformAuditor.get() != null;
         assert transformCheckpointService.get() != null;
 
-        return Collections.singletonList(
+        return Collections
-            new TransformPersistentTasksExecutor(client,
+            .singletonList(
+                new TransformPersistentTasksExecutor(
+                    client,
                     transformConfigManager.get(),
                     transformCheckpointService.get(),
                     schedulerEngine.get(),
                     transformAuditor.get(),
                     threadPool,
                     clusterService,
-                settingsModule.getSettings()));
+                    settingsModule.getSettings()
+                )
+            );
     }
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Collections.singletonList(TransformTask.NUM_FAILURE_RETRIES_SETTING);
+        return Collections.singletonList(NUM_FAILURE_RETRIES_SETTING);
     }
 
     @Override

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java

@@ -51,8 +51,7 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
         private TransformCheckpoint nextCheckpoint;
         private TransformCheckpoint sourceCheckpoint;
 
-        TransformCheckpointingInfoBuilder() {
+        TransformCheckpointingInfoBuilder() {}
-        }
 
         TransformCheckpointingInfo build() {
             if (lastCheckpoint == null) {
@@ -70,11 +69,22 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
             long nextCheckpointNumber = nextCheckpoint.getCheckpoint() > 0 ? nextCheckpoint.getCheckpoint() : 0;
 
             return new TransformCheckpointingInfo(
-                new TransformCheckpointStats(lastCheckpointNumber, null, null,
+                new TransformCheckpointStats(
-                    lastCheckpoint.getTimestamp(), lastCheckpoint.getTimeUpperBound()),
+                    lastCheckpointNumber,
-                new TransformCheckpointStats(nextCheckpointNumber, nextCheckpointPosition,
+                    null,
-                    nextCheckpointProgress, nextCheckpoint.getTimestamp(), nextCheckpoint.getTimeUpperBound()),
+                    null,
-                TransformCheckpoint.getBehind(lastCheckpoint, sourceCheckpoint));
+                    lastCheckpoint.getTimestamp(),
+                    lastCheckpoint.getTimeUpperBound()
+                ),
+                new TransformCheckpointStats(
+                    nextCheckpointNumber,
+                    nextCheckpointPosition,
+                    nextCheckpointProgress,
+                    nextCheckpoint.getTimestamp(),
+                    nextCheckpoint.getTimeUpperBound()
+                ),
+                TransformCheckpoint.getBehind(lastCheckpoint, sourceCheckpoint)
+            );
         }
 
         public TransformCheckpointingInfoBuilder setLastCheckpoint(TransformCheckpoint lastCheckpoint) {
@@ -110,10 +120,12 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
     protected final TransformAuditor transformAuditor;
     protected final TransformConfig transformConfig;
 
-    public DefaultCheckpointProvider(final Client client,
+    public DefaultCheckpointProvider(
+        final Client client,
         final TransformConfigManager transformConfigManager,
         final TransformAuditor transformAuditor,
-                                     final TransformConfig transformConfig) {
+        final TransformConfig transformConfig
+    ) {
         this.client = client;
         this.transformConfigManager = transformConfigManager;
         this.transformAuditor = transformAuditor;
@@ -126,14 +138,17 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
     }
 
     @Override
-    public void createNextCheckpoint(final TransformCheckpoint lastCheckpoint,
+    public void createNextCheckpoint(final TransformCheckpoint lastCheckpoint, final ActionListener<TransformCheckpoint> listener) {
-                              final ActionListener<TransformCheckpoint> listener) {
         final long timestamp = System.currentTimeMillis();
-        final long checkpoint = lastCheckpoint != null ? lastCheckpoint.getCheckpoint() + 1 : 1;
+        final long checkpoint = TransformCheckpoint.isNullOrEmpty(lastCheckpoint) ? 1 : lastCheckpoint.getCheckpoint() + 1;
 
         getIndexCheckpoints(ActionListener.wrap(checkpointsByIndex -> {
-            reportSourceIndexChanges(lastCheckpoint != null ? lastCheckpoint.getIndicesCheckpoints().keySet() : Collections.emptySet(),
+            reportSourceIndexChanges(
-                                     checkpointsByIndex.keySet());
+                TransformCheckpoint.isNullOrEmpty(lastCheckpoint)
+                    ? Collections.emptySet()
+                    : lastCheckpoint.getIndicesCheckpoints().keySet(),
+                checkpointsByIndex.keySet()
+            );
 
             listener.onResponse(new TransformCheckpoint(transformConfig.getId(), timestamp, checkpoint, checkpointsByIndex, 0L));
         }, listener::onFailure));
@@ -146,34 +161,41 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
             .features(new GetIndexRequest.Feature[0])
             .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
 
-        ClientHelper.executeWithHeadersAsync(transformConfig.getHeaders(), ClientHelper.TRANSFORM_ORIGIN, client, GetIndexAction.INSTANCE,
+        ClientHelper
-                getIndexRequest, ActionListener.wrap(getIndexResponse -> {
+            .executeWithHeadersAsync(
+                transformConfig.getHeaders(),
+                ClientHelper.TRANSFORM_ORIGIN,
+                client,
+                GetIndexAction.INSTANCE,
+                getIndexRequest,
+                ActionListener.wrap(getIndexResponse -> {
                     Set<String> userIndices = getIndexResponse.getIndices() != null
                         ? new HashSet<>(Arrays.asList(getIndexResponse.getIndices()))
                         : Collections.emptySet();
                     // 2nd get stats request
-                    ClientHelper.executeAsyncWithOrigin(client,
+                    ClientHelper
+                        .executeAsyncWithOrigin(
+                            client,
                             ClientHelper.TRANSFORM_ORIGIN,
                             IndicesStatsAction.INSTANCE,
                             new IndicesStatsRequest()
                                 .indices(transformConfig.getSource().getIndex())
                                 .clear()
                                 .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN),
-                        ActionListener.wrap(
+                            ActionListener.wrap(response -> {
-                            response -> {
                                 if (response.getFailedShards() != 0) {
-                                    listener.onFailure(
+                                    listener
-                                        new CheckpointException("Source has [" + response.getFailedShards() + "] failed shards"));
+                                        .onFailure(
+                                            new CheckpointException("Source has [" + response.getFailedShards() + "] failed shards")
+                                        );
                                     return;
                                 }
 
                                 listener.onResponse(extractIndexCheckPoints(response.getShards(), userIndices));
-                            },
+                            }, e -> listener.onFailure(new CheckpointException("Failed to create checkpoint", e)))
-                            e-> listener.onFailure(new CheckpointException("Failed to create checkpoint", e))
+                        );
-                        ));
+                }, e -> listener.onFailure(new CheckpointException("Failed to create checkpoint", e)))
-                },
+            );
-                e -> listener.onFailure(new CheckpointException("Failed to create checkpoint", e))
-            ));
     }
 
     static Map<String, long[]> extractIndexCheckPoints(ShardStats[] shards, Set<String> userIndices) {
@@ -220,23 +242,29 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
         // create the final structure
         Map<String, long[]> checkpointsByIndexReduced = new TreeMap<>();
 
-        checkpointsByIndex.forEach((indexName, checkpoints) -> {
+        checkpointsByIndex
+            .forEach(
+                (indexName, checkpoints) -> {
                     checkpointsByIndexReduced.put(indexName, checkpoints.values().stream().mapToLong(l -> l).toArray());
-        });
+                }
+            );
 
         return checkpointsByIndexReduced;
     }
 
     @Override
-    public void getCheckpointingInfo(TransformCheckpoint lastCheckpoint,
+    public void getCheckpointingInfo(
+        TransformCheckpoint lastCheckpoint,
         TransformCheckpoint nextCheckpoint,
         TransformIndexerPosition nextCheckpointPosition,
         TransformProgress nextCheckpointProgress,
-                                     ActionListener<TransformCheckpointingInfo> listener) {
+        ActionListener<TransformCheckpointingInfo> listener
+    ) {
 
         TransformCheckpointingInfoBuilder checkpointingInfoBuilder = new TransformCheckpointingInfoBuilder();
 
-        checkpointingInfoBuilder.setLastCheckpoint(lastCheckpoint)
+        checkpointingInfoBuilder
+            .setLastCheckpoint(lastCheckpoint)
             .setNextCheckpoint(nextCheckpoint)
             .setNextCheckpointPosition(nextCheckpointPosition)
             .setNextCheckpointProgress(nextCheckpointProgress);
@@ -244,16 +272,19 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
         long timestamp = System.currentTimeMillis();
 
         getIndexCheckpoints(ActionListener.wrap(checkpointsByIndex -> {
-            checkpointingInfoBuilder.setSourceCheckpoint(
+            checkpointingInfoBuilder
-                    new TransformCheckpoint(transformConfig.getId(), timestamp, -1L, checkpointsByIndex, 0L));
+                .setSourceCheckpoint(new TransformCheckpoint(transformConfig.getId(), timestamp, -1L, checkpointsByIndex, 0L));
             listener.onResponse(checkpointingInfoBuilder.build());
         }, listener::onFailure));
     }
 
     @Override
-    public void getCheckpointingInfo(long lastCheckpointNumber, TransformIndexerPosition nextCheckpointPosition,
+    public void getCheckpointingInfo(
+        long lastCheckpointNumber,
+        TransformIndexerPosition nextCheckpointPosition,
         TransformProgress nextCheckpointProgress,
-                                     ActionListener<TransformCheckpointingInfo> listener) {
+        ActionListener<TransformCheckpointingInfo> listener
+    ) {
 
         TransformCheckpointingInfoBuilder checkpointingInfoBuilder = new TransformCheckpointingInfoBuilder();
 
@@ -262,47 +293,56 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
         long timestamp = System.currentTimeMillis();
 
         // <3> got the source checkpoint, notify the user
-        ActionListener<Map<String, long[]>> checkpointsByIndexListener = ActionListener.wrap(
+        ActionListener<Map<String, long[]>> checkpointsByIndexListener = ActionListener.wrap(checkpointsByIndex -> {
-                checkpointsByIndex -> {
+            checkpointingInfoBuilder
-                    checkpointingInfoBuilder.setSourceCheckpoint(
+                .setSourceCheckpoint(new TransformCheckpoint(transformConfig.getId(), timestamp, -1L, checkpointsByIndex, 0L));
-                        new TransformCheckpoint(transformConfig.getId(), timestamp, -1L, checkpointsByIndex, 0L));
             listener.onResponse(checkpointingInfoBuilder.build());
-                },
+        }, e -> {
-                e -> {
+            logger
-                    logger.debug((Supplier<?>) () -> new ParameterizedMessage(
+                .debug(
-                            "Failed to retrieve source checkpoint for transform [{}]", transformConfig.getId()), e);
+                    (Supplier<?>) () -> new ParameterizedMessage(
-                    listener.onFailure(new CheckpointException("Failure during source checkpoint info retrieval", e));
+                        "[{}] failed to retrieve source checkpoint for transform",
-                }
+                        transformConfig.getId()
+                    ),
+                    e
                 );
+            listener.onFailure(new CheckpointException("Failure during source checkpoint info retrieval", e));
+        });
 
         // <2> got the next checkpoint, get the source checkpoint
-        ActionListener<TransformCheckpoint> nextCheckpointListener = ActionListener.wrap(
+        ActionListener<TransformCheckpoint> nextCheckpointListener = ActionListener.wrap(nextCheckpointObj -> {
-                nextCheckpointObj -> {
             checkpointingInfoBuilder.setNextCheckpoint(nextCheckpointObj);
             getIndexCheckpoints(checkpointsByIndexListener);
-                },
+        }, e -> {
-                e -> {
+            logger
-                    logger.debug((Supplier<?>) () -> new ParameterizedMessage(
+                .debug(
-                            "Failed to retrieve next checkpoint [{}] for transform [{}]", lastCheckpointNumber + 1,
+                    (Supplier<?>) () -> new ParameterizedMessage(
-                            transformConfig.getId()), e);
+                        "[{}] failed to retrieve next checkpoint [{}]",
-                    listener.onFailure(new CheckpointException("Failure during next checkpoint info retrieval", e));
+                        transformConfig.getId(),
-                }
+                        lastCheckpointNumber + 1
+
+                    ),
+                    e
                 );
+            listener.onFailure(new CheckpointException("Failure during next checkpoint info retrieval", e));
+        });
 
         // <1> got last checkpoint, get the next checkpoint
-        ActionListener<TransformCheckpoint> lastCheckpointListener = ActionListener.wrap(
+        ActionListener<TransformCheckpoint> lastCheckpointListener = ActionListener.wrap(lastCheckpointObj -> {
-            lastCheckpointObj -> {
             checkpointingInfoBuilder.lastCheckpoint = lastCheckpointObj;
-                transformConfigManager.getTransformCheckpoint(transformConfig.getId(), lastCheckpointNumber + 1,
+            transformConfigManager.getTransformCheckpoint(transformConfig.getId(), lastCheckpointNumber + 1, nextCheckpointListener);
-                        nextCheckpointListener);
+        }, e -> {
-            },
+            logger
-            e -> {
+                .debug(
-                logger.debug((Supplier<?>) () -> new ParameterizedMessage(
+                    (Supplier<?>) () -> new ParameterizedMessage(
-                        "Failed to retrieve last checkpoint [{}] for transform [{}]", lastCheckpointNumber,
+                        "[{}] failed to retrieve last checkpoint [{}]",
-                        transformConfig.getId()), e);
+                        transformConfig.getId(),
-                listener.onFailure(new CheckpointException("Failure during last checkpoint info retrieval", e));
+                        lastCheckpointNumber
-            }
+                    ),
+                    e
                 );
+            listener.onFailure(new CheckpointException("Failure during last checkpoint info retrieval", e));
+        });
 
         if (lastCheckpointNumber != 0) {
             transformConfigManager.getTransformCheckpoint(transformConfig.getId(), lastCheckpointNumber, lastCheckpointListener);
@@ -321,20 +361,25 @@ public class DefaultCheckpointProvider implements CheckpointProvider {
         // spam protection: only warn the first time
         if (newSourceIndexes.isEmpty() && lastSourceIndexes.isEmpty() == false) {
             String message = "Source did not resolve to any open indexes";
-            logger.warn("{} for transform [{}]", message, transformConfig.getId());
+            logger.warn("[{}] {}", transformConfig.getId(), message);
             transformAuditor.warning(transformConfig.getId(), message);
         } else {
             Set<String> removedIndexes = Sets.difference(lastSourceIndexes, newSourceIndexes);
             Set<String> addedIndexes = Sets.difference(newSourceIndexes, lastSourceIndexes);
 
             if (removedIndexes.size() + addedIndexes.size() > AUDIT_CONCRETED_SOURCE_INDEX_CHANGES) {
-                String message = "Source index resolve found more than " + AUDIT_CONCRETED_SOURCE_INDEX_CHANGES + " changes, ["
+                String message = "Source index resolve found more than "
-                        + removedIndexes.size() + "] removed indexes, [" + addedIndexes.size() + "] new indexes";
+                    + AUDIT_CONCRETED_SOURCE_INDEX_CHANGES
-                logger.debug("{} for transform [{}]", message, transformConfig.getId());
+                    + " changes, ["
+                    + removedIndexes.size()
+                    + "] removed indexes, ["
+                    + addedIndexes.size()
+                    + "] new indexes";
+                logger.debug("[{}] {}", transformConfig.getId(), message);
                 transformAuditor.info(transformConfig.getId(), message);
             } else if (removedIndexes.size() + addedIndexes.size() > 0) {
                 String message = "Source index resolve found changes, removedIndexes: " + removedIndexes + ", new indexes: " + addedIndexes;
-                logger.debug("{} for transform [{}]", message, transformConfig.getId());
+                logger.debug("[{}] {}", transformConfig.getId(), message);
                 transformAuditor.info(transformConfig.getId(), message);
             }
         }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProvider.java

@@ -18,9 +18,9 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
-import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
 import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
 
@@ -30,17 +30,18 @@ public class TimeBasedCheckpointProvider extends DefaultCheckpointProvider {
 
     private final TimeSyncConfig timeSyncConfig;
 
-    TimeBasedCheckpointProvider(final Client client,
+    TimeBasedCheckpointProvider(
+        final Client client,
         final TransformConfigManager transformConfigManager,
         final TransformAuditor transformAuditor,
-                                final TransformConfig transformConfig) {
+        final TransformConfig transformConfig
+    ) {
         super(client, transformConfigManager, transformAuditor, transformConfig);
         timeSyncConfig = (TimeSyncConfig) transformConfig.getSyncConfig();
     }
 
     @Override
-    public void sourceHasChanged(TransformCheckpoint lastCheckpoint,
+    public void sourceHasChanged(TransformCheckpoint lastCheckpoint, ActionListener<Boolean> listener) {
-            ActionListener<Boolean> listener) {
 
         final long timestamp = getTime();
 
@@ -53,36 +54,51 @@ public class TimeBasedCheckpointProvider extends DefaultCheckpointProvider {
             .trackTotalHitsUpTo(1);
 
         QueryBuilder queryBuilder = transformConfig.getSource().getQueryConfig().getQuery();
-        BoolQueryBuilder filteredQuery = new BoolQueryBuilder().
+        BoolQueryBuilder filteredQuery = new BoolQueryBuilder()
-                filter(queryBuilder).
+            .filter(queryBuilder)
-                filter(new RangeQueryBuilder(timeSyncConfig.getField()).
+            .filter(
-                        gte(lastCheckpoint.getTimeUpperBound()).
+                new RangeQueryBuilder(timeSyncConfig.getField())
-                        lt(timestamp - timeSyncConfig.getDelay().millis()).format("epoch_millis"));
+                    .gte(lastCheckpoint.getTimeUpperBound())
+                    .lt(timestamp - timeSyncConfig.getDelay().millis())
+                    .format("epoch_millis")
+            );
 
         sourceBuilder.query(filteredQuery);
         searchRequest.source(sourceBuilder);
 
         logger.trace("query for changes based on time: {}", sourceBuilder);
 
-        ClientHelper.executeWithHeadersAsync(transformConfig.getHeaders(), ClientHelper.TRANSFORM_ORIGIN, client, SearchAction.INSTANCE,
+        ClientHelper
-                searchRequest, ActionListener.wrap(r -> {
+            .executeWithHeadersAsync(
-                    listener.onResponse(r.getHits().getTotalHits().value > 0L);
+                transformConfig.getHeaders(),
-                }, listener::onFailure));
+                ClientHelper.TRANSFORM_ORIGIN,
+                client,
+                SearchAction.INSTANCE,
+                searchRequest,
+                ActionListener.wrap(r -> { listener.onResponse(r.getHits().getTotalHits().value > 0L); }, listener::onFailure)
+            );
     }
 
     @Override
-    public void createNextCheckpoint(final TransformCheckpoint lastCheckpoint,
+    public void createNextCheckpoint(final TransformCheckpoint lastCheckpoint, final ActionListener<TransformCheckpoint> listener) {
-            final ActionListener<TransformCheckpoint> listener) {
         final long timestamp = getTime();
-        final long checkpoint = lastCheckpoint != null ? lastCheckpoint.getCheckpoint() + 1 : 1;
+        final long checkpoint = TransformCheckpoint.isNullOrEmpty(lastCheckpoint) ? 1 : lastCheckpoint.getCheckpoint() + 1;
 
         // for time based synchronization
         long timeUpperBound = timestamp - timeSyncConfig.getDelay().millis();
 
-        getIndexCheckpoints(ActionListener.wrap(checkpointsByIndex -> {
+        getIndexCheckpoints(
-            listener.onResponse(
+            ActionListener
-                    new TransformCheckpoint(transformConfig.getId(), timestamp, checkpoint, checkpointsByIndex, timeUpperBound));
+                .wrap(
-        }, listener::onFailure));
+                    checkpointsByIndex -> {
+                        listener
+                            .onResponse(
+                                new TransformCheckpoint(transformConfig.getId(), timestamp, checkpoint, checkpointsByIndex, timeUpperBound)
+                            );
+                    },
+                    listener::onFailure
+                )
+        );
     }
 
     // for the purpose of testing

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/SeqNoPrimaryTermAndIndex.java

@@ -27,7 +27,7 @@ public class SeqNoPrimaryTermAndIndex {
         return new SeqNoPrimaryTermAndIndex(response.getSeqNo(), response.getPrimaryTerm(), response.getIndex());
     }
 
-    SeqNoPrimaryTermAndIndex(long seqNo, long primaryTerm, String index) {
+    public SeqNoPrimaryTermAndIndex(long seqNo, long primaryTerm, String index) {
         this.seqNo = seqNo;
         this.primaryTerm = primaryTerm;
         this.index = index;
@@ -61,9 +61,7 @@ public class SeqNoPrimaryTermAndIndex {
         }
 
         SeqNoPrimaryTermAndIndex other = (SeqNoPrimaryTermAndIndex) obj;
-        return Objects.equals(seqNo, other.seqNo)
+        return Objects.equals(seqNo, other.seqNo) && Objects.equals(primaryTerm, other.primaryTerm) && Objects.equals(index, other.index);
-            && Objects.equals(primaryTerm, other.primaryTerm)
-            && Objects.equals(index, other.index);
     }
 
     @Override

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java

@@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -20,16 +19,14 @@ import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
-import org.elasticsearch.xpack.core.transform.TransformMessages;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformProgress;
 import org.elasticsearch.xpack.core.transform.transforms.TransformState;
 import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc;
@@ -37,38 +34,28 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
 import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
-import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
 import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex;
-import org.elasticsearch.xpack.transform.transforms.pivot.AggregationResultUtils;
+import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
 
-import java.time.Instant;
 import java.util.Map;
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 class ClientTransformIndexer extends TransformIndexer {
 
     private static final Logger logger = LogManager.getLogger(ClientTransformIndexer.class);
 
-    private long logEvery = 1;
-    private long logCount = 0;
     private final Client client;
-    private final TransformConfigManager transformsConfigManager;
-    private final CheckpointProvider checkpointProvider;
-    private final TransformTask transformTask;
-    private final AtomicInteger failureCount;
-    private volatile boolean auditBulkFailures = true;
-    // Indicates that the source has changed for the current run
-    private volatile boolean hasSourceChanged = true;
-    // Keeps track of the last exception that was written to our audit, keeps us from spamming the audit index
-    private volatile String lastAuditedExceptionMessage = null;
     private final AtomicBoolean oldStatsCleanedUp = new AtomicBoolean(false);
-    private volatile boolean shouldStopAtCheckpoint = false;
-    private volatile Instant changesLastDetectedAt;
 
-    ClientTransformIndexer(TransformConfigManager transformsConfigManager,
+    private final AtomicReference<SeqNoPrimaryTermAndIndex> seqNoPrimaryTermAndIndex;
+
+    ClientTransformIndexer(
+        Executor executor,
+        TransformConfigManager transformsConfigManager,
         CheckpointProvider checkpointProvider,
+        TransformProgressGatherer progressGatherer,
         AtomicReference<IndexerState> initialState,
         TransformIndexerPosition initialPosition,
         Client client,
@@ -79,12 +66,16 @@ class ClientTransformIndexer extends TransformIndexer {
         TransformProgress transformProgress,
         TransformCheckpoint lastCheckpoint,
         TransformCheckpoint nextCheckpoint,
-                           TransformTask parentTask,
+        SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex,
-                           boolean shouldStopAtCheckpoint) {
+        TransformContext context,
-        super(ExceptionsHelper.requireNonNull(parentTask, "parentTask")
+        boolean shouldStopAtCheckpoint
-                .getThreadPool()
+    ) {
-                .executor(ThreadPool.Names.GENERIC),
+        super(
-            ExceptionsHelper.requireNonNull(auditor, "auditor"),
+            ExceptionsHelper.requireNonNull(executor, "executor"),
+            transformsConfigManager,
+            checkpointProvider,
+            progressGatherer,
+            auditor,
             transformConfig,
             fieldMappings,
             ExceptionsHelper.requireNonNull(initialState, "initialState"),
@@ -92,210 +83,72 @@ class ClientTransformIndexer extends TransformIndexer {
             initialStats == null ? new TransformIndexerStats() : initialStats,
             transformProgress,
             lastCheckpoint,
-            nextCheckpoint);
+            nextCheckpoint,
-        this.transformsConfigManager = ExceptionsHelper.requireNonNull(transformsConfigManager, "transformsConfigManager");
+            context
-        this.checkpointProvider = ExceptionsHelper.requireNonNull(checkpointProvider, "checkpointProvider");
+        );
-
         this.client = ExceptionsHelper.requireNonNull(client, "client");
-        this.transformTask = parentTask;
+        this.seqNoPrimaryTermAndIndex = new AtomicReference<>(seqNoPrimaryTermAndIndex);
-        this.failureCount = new AtomicInteger(0);
-        this.shouldStopAtCheckpoint = shouldStopAtCheckpoint;
-    }
 
-    boolean shouldStopAtCheckpoint() {
+        // TODO: move into context constructor
-        return shouldStopAtCheckpoint;
+        context.setShouldStopAtCheckpoint(shouldStopAtCheckpoint);
-    }
-
-    void setShouldStopAtCheckpoint(boolean shouldStopAtCheckpoint) {
-        this.shouldStopAtCheckpoint = shouldStopAtCheckpoint;
     }
 
     void persistShouldStopAtCheckpoint(boolean shouldStopAtCheckpoint, ActionListener<Void> shouldStopAtCheckpointListener) {
-        if (this.shouldStopAtCheckpoint == shouldStopAtCheckpoint ||
+        if (context.shouldStopAtCheckpoint() == shouldStopAtCheckpoint
-            getState() == IndexerState.STOPPED ||
+            || getState() == IndexerState.STOPPED
-            getState() == IndexerState.STOPPING) {
+            || getState() == IndexerState.STOPPING) {
             shouldStopAtCheckpointListener.onResponse(null);
             return;
         }
         TransformState state = new TransformState(
-            transformTask.getTaskState(),
+            context.getTaskState(),
             getState(),
             getPosition(),
-            transformTask.getCheckpoint(),
+            context.getCheckpoint(),
-            transformTask.getStateReason(),
+            context.getStateReason(),
             getProgress(),
             null, // Node attributes
-            shouldStopAtCheckpoint);
+            shouldStopAtCheckpoint
-        doSaveState(state,
+        );
-            ActionListener.wrap(
+        doSaveState(state, ActionListener.wrap(r -> {
-                r -> {
             // We only want to update this internal value if it is persisted as such
-                    this.shouldStopAtCheckpoint = shouldStopAtCheckpoint;
+            context.setShouldStopAtCheckpoint(shouldStopAtCheckpoint);
-                    logger.debug("[{}] successfully persisted should_stop_at_checkpoint update [{}]",
+            logger.debug("[{}] successfully persisted should_stop_at_checkpoint update [{}]", getJobId(), shouldStopAtCheckpoint);
-                        getJobId(),
-                        shouldStopAtCheckpoint);
             shouldStopAtCheckpointListener.onResponse(null);
-                },
+        }, statsExc -> {
-                statsExc -> {
+            logger.warn("[{}] failed to persist should_stop_at_checkpoint update [{}]", getJobId(), shouldStopAtCheckpoint);
-                    logger.warn("[{}] failed to persist should_stop_at_checkpoint update [{}]",
-                        getJobId(),
-                        shouldStopAtCheckpoint);
             shouldStopAtCheckpointListener.onFailure(statsExc);
-                }
+        }));
-            ));
-    }
-
-    @Override
-    protected void onStart(long now, ActionListener<Boolean> listener) {
-        if (transformTask.getTaskState() == TransformTaskState.FAILED) {
-            logger.debug("[{}] attempted to start while failed.", getJobId());
-            listener.onFailure(new ElasticsearchException("Attempted to start a failed transform [{}].", getJobId()));
-            return;
-        }
-        // On each run, we need to get the total number of docs and reset the count of processed docs
-        // Since multiple checkpoints can be executed in the task while it is running on the same node, we need to gather
-        // the progress here, and not in the executor.
-        ActionListener<Void> updateConfigListener = ActionListener.wrap(
-            updateConfigResponse -> {
-                if (initialRun()) {
-                    createCheckpoint(ActionListener.wrap(cp -> {
-                        nextCheckpoint = cp;
-                        // If nextCheckpoint > 1, this means that we are now on the checkpoint AFTER the batch checkpoint
-                        // Consequently, the idea of percent complete no longer makes sense.
-                        if (nextCheckpoint.getCheckpoint() > 1) {
-                            progress = new TransformProgress(null, 0L, 0L);
-                            super.onStart(now, listener);
-                            return;
-                        }
-                        TransformProgressGatherer.getInitialProgress(this.client, buildFilterQuery(), getConfig(), ActionListener.wrap(
-                            newProgress -> {
-                                logger.trace("[{}] reset the progress from [{}] to [{}].", getJobId(), progress, newProgress);
-                                progress = newProgress;
-                                super.onStart(now, listener);
-                            },
-                            failure -> {
-                                progress = null;
-                                logger.warn(new ParameterizedMessage("[{}] unable to load progress information for task.",
-                                    getJobId()),
-                                    failure);
-                                super.onStart(now, listener);
-                            }
-                        ));
-                    }, listener::onFailure));
-                } else {
-                    super.onStart(now, listener);
-                }
-            },
-            listener::onFailure
-        );
-
-        // If we are continuous, we will want to verify we have the latest stored configuration
-        ActionListener<Void> changedSourceListener = ActionListener.wrap(
-            r -> {
-                if (isContinuous()) {
-                    transformsConfigManager.getTransformConfiguration(getJobId(), ActionListener.wrap(
-                        config -> {
-                            transformConfig = config;
-                            logger.debug("[{}] successfully refreshed transform config from index.", getJobId());
-                            updateConfigListener.onResponse(null);
-                        },
-                        failure -> {
-                            String msg = TransformMessages.getMessage(
-                                TransformMessages.FAILED_TO_RELOAD_TRANSFORM_CONFIGURATION,
-                                getJobId());
-                            logger.error(msg, failure);
-                            // If the transform config index or the transform config is gone, something serious occurred
-                            // We are in an unknown state and should fail out
-                            if (failure instanceof ResourceNotFoundException) {
-                                updateConfigListener.onFailure(new TransformConfigReloadingException(msg, failure));
-                            } else {
-                                auditor.warning(getJobId(), msg);
-                                updateConfigListener.onResponse(null);
-                            }
-                        }
-                    ));
-                } else {
-                    updateConfigListener.onResponse(null);
-                }
-            },
-            listener::onFailure
-        );
-
-        // If we are not on the initial batch checkpoint and its the first pass of whatever continuous checkpoint we are on,
-        // we should verify if there are local changes based on the sync config. If not, do not proceed further and exit.
-        if (transformTask.getCheckpoint() > 0 && initialRun()) {
-            sourceHasChanged(ActionListener.wrap(
-                hasChanged -> {
-                    hasSourceChanged = hasChanged;
-                    if (hasChanged) {
-                        changesLastDetectedAt = Instant.now();
-                        logger.debug("[{}] source has changed, triggering new indexer run.", getJobId());
-                        changedSourceListener.onResponse(null);
-                    } else {
-                        logger.trace("[{}] source has not changed, finish indexer early.", getJobId());
-                        // No changes, stop executing
-                        listener.onResponse(false);
-                    }
-                },
-                failure -> {
-                    // If we failed determining if the source changed, it's safer to assume there were changes.
-                    // We should allow the failure path to complete as normal
-                    hasSourceChanged = true;
-                    listener.onFailure(failure);
-                }
-            ));
-        } else {
-            hasSourceChanged = true;
-            changedSourceListener.onResponse(null);
-        }
-    }
-
-    public CheckpointProvider getCheckpointProvider() {
-        return checkpointProvider;
-    }
-
-    Instant getChangesLastDetectedAt() {
-        return changesLastDetectedAt;
-    }
-
-    @Override
-    public synchronized boolean maybeTriggerAsyncJob(long now) {
-        if (transformTask.getTaskState() == TransformTaskState.FAILED) {
-            logger.debug("[{}] schedule was triggered for transform but task is failed. Ignoring trigger.", getJobId());
-            return false;
-        }
-
-        // ignore trigger if indexer is running, prevents log spam in A2P indexer
-        IndexerState indexerState = getState();
-        if (IndexerState.INDEXING.equals(indexerState) || IndexerState.STOPPING.equals(indexerState)) {
-            logger.debug("[{}] indexer for transform has state [{}]. Ignoring trigger.", getJobId(), indexerState);
-            return false;
-        }
-
-        return super.maybeTriggerAsyncJob(now);
     }
 
     @Override
     protected void doNextSearch(SearchRequest request, ActionListener<SearchResponse> nextPhase) {
-        if (transformTask.getTaskState() == TransformTaskState.FAILED) {
+        if (context.getTaskState() == TransformTaskState.FAILED) {
             logger.debug("[{}] attempted to search while failed.", getJobId());
-            nextPhase.onFailure(new ElasticsearchException("Attempted to do a search request for failed transform [{}].",
+            nextPhase.onFailure(new ElasticsearchException("Attempted to do a search request for failed transform [{}].", getJobId()));
-                getJobId()));
             return;
         }
-        ClientHelper.executeWithHeadersAsync(transformConfig.getHeaders(), ClientHelper.TRANSFORM_ORIGIN, client,
+        ClientHelper
-                SearchAction.INSTANCE, request, nextPhase);
+            .executeWithHeadersAsync(
+                transformConfig.getHeaders(),
+                ClientHelper.TRANSFORM_ORIGIN,
+                client,
+                SearchAction.INSTANCE,
+                request,
+                nextPhase
+            );
     }
 
     @Override
     protected void doNextBulk(BulkRequest request, ActionListener<BulkResponse> nextPhase) {
-        if (transformTask.getTaskState() == TransformTaskState.FAILED) {
+        if (context.getTaskState() == TransformTaskState.FAILED) {
             logger.debug("[{}] attempted to bulk index while failed.", getJobId());
-            nextPhase.onFailure(new ElasticsearchException("Attempted to do a bulk index request for failed transform [{}].",
+            nextPhase.onFailure(new ElasticsearchException("Attempted to do a bulk index request for failed transform [{}].", getJobId()));
-                getJobId()));
             return;
         }
-        ClientHelper.executeWithHeadersAsync(transformConfig.getHeaders(),
+        ClientHelper
+            .executeWithHeadersAsync(
+                transformConfig.getHeaders(),
                 ClientHelper.TRANSFORM_ORIGIN,
                 client,
                 BulkAction.INSTANCE,
@@ -310,28 +163,37 @@ class ClientTransformIndexer extends TransformIndexer {
                             // TODO gather information on irrecoverable failures and update isIrrecoverableFailure
                         }
                         if (auditBulkFailures) {
-                        auditor.warning(getJobId(),
+                            String failureMessage = bulkResponse.buildFailureMessage();
-                            "Experienced at least [" +
+                            logger.debug("[{}] Bulk index failure encountered: {}", getJobId(), failureMessage);
-                                failureCount +
+                            auditor
-                                "] bulk index failures. See the logs of the node running the transform for details. " +
+                                .warning(
-                                bulkResponse.buildFailureMessage());
+                                    getJobId(),
+                                    "Experienced at least ["
+                                        + failureCount
+                                        + "] bulk index failures. See the logs of the node running the transform for details. "
+                                        + failureMessage
+                                );
                             auditBulkFailures = false;
                         }
                         // This calls AsyncTwoPhaseIndexer#finishWithIndexingFailure
                         // It increments the indexing failure, and then calls the `onFailure` logic
-                    nextPhase.onFailure(
+                        nextPhase
-                        new BulkIndexingException("Bulk index experienced failures. " +
+                            .onFailure(
-                            "See the logs of the node running the transform for details."));
+                                new BulkIndexingException(
+                                    "Bulk index experienced failures. " + "See the logs of the node running the transform for details."
+                                )
+                            );
                     } else {
                         auditBulkFailures = true;
                         nextPhase.onResponse(bulkResponse);
                     }
-            }, nextPhase::onFailure));
+                }, nextPhase::onFailure)
+            );
     }
 
     @Override
     protected void doSaveState(IndexerState indexerState, TransformIndexerPosition position, Runnable next) {
-        if (transformTask.getTaskState() == TransformTaskState.FAILED) {
+        if (context.getTaskState() == TransformTaskState.FAILED) {
             logger.debug("[{}] attempted to save state and stats while failed.", getJobId());
             // If we are failed, we should call next to allow failure handling to occur if necessary.
             next.run();
@@ -343,7 +205,7 @@ class ClientTransformIndexer extends TransformIndexer {
             return;
         }
 
-        boolean shouldStopAtCheckpoint = shouldStopAtCheckpoint();
+        boolean shouldStopAtCheckpoint = context.shouldStopAtCheckpoint();
 
         // If we should stop at the next checkpoint, are STARTED, and with `initialRun()` we are in one of two states
         // 1. We have just called `onFinish` completing our request, but `shouldStopAtCheckpoint` was set to `true` before our check
@@ -354,8 +216,7 @@ class ClientTransformIndexer extends TransformIndexer {
         if (shouldStopAtCheckpoint && initialRun() && indexerState.equals(IndexerState.STARTED)) {
             indexerState = IndexerState.STOPPED;
             auditor.info(transformConfig.getId(), "Transform is no longer in the middle of a checkpoint, initiating stop.");
-            logger.info("[{}] transform is no longer in the middle of a checkpoint, initiating stop.",
+            logger.info("[{}] transform is no longer in the middle of a checkpoint, initiating stop.", transformConfig.getId());
-                transformConfig.getId());
         }
 
         // This means that the indexer was triggered to discover changes, found none, and exited early.
@@ -366,11 +227,9 @@ class ClientTransformIndexer extends TransformIndexer {
             return;
         }
 
-        TransformTaskState taskState = transformTask.getTaskState();
+        TransformTaskState taskState = context.getTaskState();
 
-        if (indexerState.equals(IndexerState.STARTED)
+        if (indexerState.equals(IndexerState.STARTED) && context.getCheckpoint() == 1 && this.isContinuous() == false) {
-            && transformTask.getCheckpoint() == 1
-            && this.isContinuous() == false) {
             // set both to stopped so they are persisted as such
             indexerState = IndexerState.STOPPED;
 
@@ -397,266 +256,73 @@ class ClientTransformIndexer extends TransformIndexer {
             taskState,
             indexerState,
             position,
-            transformTask.getCheckpoint(),
+            context.getCheckpoint(),
-            transformTask.getStateReason(),
+            context.getStateReason(),
             getProgress(),
             null,
-            shouldStopAtCheckpoint);
+            shouldStopAtCheckpoint
+        );
         logger.debug("[{}] updating persistent state of transform to [{}].", transformConfig.getId(), state.toString());
 
-        doSaveState(state, ActionListener.wrap(
+        doSaveState(state, ActionListener.wrap(r -> next.run(), e -> next.run()));
-            r -> next.run(),
-            e -> next.run()
-        ));
     }
 
     private void doSaveState(TransformState state, ActionListener<Void> listener) {
 
         // This could be `null` but the putOrUpdateTransformStoredDoc handles that case just fine
-        SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex = transformTask.getSeqNoPrimaryTermAndIndex();
+        SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex = getSeqNoPrimaryTermAndIndex();
 
         // Persist the current state and stats in the internal index. The interval of this method being
         // called is controlled by AsyncTwoPhaseIndexer#onBulkResponse which calls doSaveState every so
         // often when doing bulk indexing calls or at the end of one indexing run.
-        transformsConfigManager.putOrUpdateTransformStoredDoc(
+        transformsConfigManager
+            .putOrUpdateTransformStoredDoc(
                 new TransformStoredDoc(getJobId(), state, getStats()),
                 seqNoPrimaryTermAndIndex,
-                ActionListener.wrap(
+                ActionListener.wrap(r -> {
-                        r -> {
+                    updateSeqNoPrimaryTermAndIndex(seqNoPrimaryTermAndIndex, r);
-                            transformTask.updateSeqNoPrimaryTermAndIndex(seqNoPrimaryTermAndIndex, r);
                     // for auto stop shutdown the task
                     if (state.getTaskState().equals(TransformTaskState.STOPPED)) {
-                                transformTask.shutdown();
+                        context.shutdown();
                     }
                     // Only do this clean up once, if it succeeded, no reason to do the query again.
                     if (oldStatsCleanedUp.compareAndSet(false, true)) {
-                                transformsConfigManager.deleteOldTransformStoredDocuments(getJobId(), ActionListener.wrap(
+                        transformsConfigManager.deleteOldTransformStoredDocuments(getJobId(), ActionListener.wrap(nil -> {
-                                    nil -> {
                             logger.trace("[{}] deleted old transform stats and state document", getJobId());
                             listener.onResponse(null);
-                                    },
+                        }, e -> {
-                                    e -> {
+                            String msg = LoggerMessageFormat.format("[{}] failed deleting old transform configurations.", getJobId());
-                                        String msg = LoggerMessageFormat.format("[{}] failed deleting old transform configurations.",
-                                            getJobId());
                             logger.warn(msg, e);
                             // If we have failed, we should attempt the clean up again later
                             oldStatsCleanedUp.set(false);
                             listener.onResponse(null);
-                                    }
+                        }));
-                                ));
                     } else {
                         listener.onResponse(null);
                     }
-                        },
+                }, statsExc -> {
-                        statsExc -> {
+                    logger.error(new ParameterizedMessage("[{}] updating stats of transform failed.", transformConfig.getId()), statsExc);
-                            logger.error(new ParameterizedMessage("[{}] updating stats of transform failed.",
+                    auditor.warning(getJobId(), "Failure updating stats of transform: " + statsExc.getMessage());
-                                transformConfig.getId()),
-                                statsExc);
-                            auditor.warning(getJobId(),
-                                "Failure updating stats of transform: " + statsExc.getMessage());
                     // for auto stop shutdown the task
                     if (state.getTaskState().equals(TransformTaskState.STOPPED)) {
-                                transformTask.shutdown();
+                        context.shutdown();
                     }
                     listener.onFailure(statsExc);
-                        }
+                })
-                ));
+            );
+
     }
 
-    @Override
+    void updateSeqNoPrimaryTermAndIndex(SeqNoPrimaryTermAndIndex expectedValue, SeqNoPrimaryTermAndIndex newValue) {
-    protected void onFailure(Exception exc) {
+        boolean updated = seqNoPrimaryTermAndIndex.compareAndSet(expectedValue, newValue);
-        // the failure handler must not throw an exception due to internal problems
+        // This should never happen. We ONLY ever update this value if at initialization or we just finished updating the document
-        try {
+        // famous last words...
-            handleFailure(exc);
+        assert updated : "[" + getJobId() + "] unexpected change to seqNoPrimaryTermAndIndex.";
-        } catch (Exception e) {
-            logger.error(
-                new ParameterizedMessage("[{}] transform encountered an unexpected internal exception: ", getJobId()),
-                e);
-        }
     }
 
-    @Override
+    @Nullable
-    protected void onFinish(ActionListener<Void> listener) {
+    SeqNoPrimaryTermAndIndex getSeqNoPrimaryTermAndIndex() {
-        try {
+        return seqNoPrimaryTermAndIndex.get();
-            // This indicates an early exit since no changes were found.
-            // So, don't treat this like a checkpoint being completed, as no work was done.
-            if (hasSourceChanged == false) {
-                if (shouldStopAtCheckpoint) {
-                    stop();
-                }
-                listener.onResponse(null);
-                return;
-            }
-            // TODO: needs cleanup super is called with a listener, but listener.onResponse is called below
-            // super.onFinish() fortunately ignores the listener
-            super.onFinish(listener);
-            long checkpoint = transformTask.incrementCheckpoint();
-            lastCheckpoint = getNextCheckpoint();
-            nextCheckpoint = null;
-            // Reset our failure count as we have finished and may start again with a new checkpoint
-            failureCount.set(0);
-            transformTask.setStateReason(null);
-
-            // With bucket_selector we could have read all the buckets and completed the transform
-            // but not "see" all the buckets since they were filtered out. Consequently, progress would
-            // show less than 100% even though we are done.
-            // NOTE: this method is called in the same thread as the processing thread.
-            // Theoretically, there should not be a race condition with updating progress here.
-            // NOTE 2: getPercentComplete should only NOT be null on the first (batch) checkpoint
-            if (progress != null && progress.getPercentComplete() != null && progress.getPercentComplete() < 100.0) {
-                progress.incrementDocsProcessed(progress.getTotalDocs() - progress.getDocumentsProcessed());
-            }
-            // If the last checkpoint is now greater than 1, that means that we have just processed the first
-            // continuous checkpoint and should start recording the exponential averages
-            if (lastCheckpoint != null && lastCheckpoint.getCheckpoint() > 1) {
-                long docsIndexed = 0;
-                long docsProcessed = 0;
-                // This should not happen as we simply create a new one when we reach continuous checkpoints
-                // but this is a paranoid `null` check
-                if (progress != null) {
-                    docsIndexed = progress.getDocumentsIndexed();
-                    docsProcessed = progress.getDocumentsProcessed();
-                }
-                long durationMs = System.currentTimeMillis() - lastCheckpoint.getTimestamp();
-                getStats().incrementCheckpointExponentialAverages(durationMs < 0 ? 0 : durationMs, docsIndexed, docsProcessed);
-            }
-            if (shouldAuditOnFinish(checkpoint)) {
-                auditor.info(getJobId(),
-                    "Finished indexing for transform checkpoint [" + checkpoint + "].");
-            }
-            logger.debug(
-                "[{}] finished indexing for transform checkpoint [{}].", getJobId(), checkpoint);
-            auditBulkFailures = true;
-            if (shouldStopAtCheckpoint) {
-                stop();
-            }
-            listener.onResponse(null);
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
-    }
-
-    /**
-     * Indicates if an audit message should be written when onFinish is called for the given checkpoint
-     * We audit the first checkpoint, and then every 10 checkpoints until completedCheckpoint == 99
-     * Then we audit every 100, until completedCheckpoint == 999
-     *
-     * Then we always audit every 1_000 checkpoints
-     *
-     * @param completedCheckpoint The checkpoint that was just completed
-     * @return {@code true} if an audit message should be written
-     */
-    protected boolean shouldAuditOnFinish(long completedCheckpoint) {
-        if (++logCount % logEvery != 0) {
-            return false;
-        }
-        if (completedCheckpoint == 0) {
-            return true;
-        }
-        int log10Checkpoint = (int) Math.floor(Math.log10(completedCheckpoint));
-        logEvery = log10Checkpoint >= 3  ? 1_000 : (int)Math.pow(10.0, log10Checkpoint);
-        logCount = 0;
-        return true;
-    }
-
-    @Override
-    protected void onStop() {
-        auditor.info(transformConfig.getId(), "Transform has stopped.");
-        logger.info("[{}] transform has stopped.", transformConfig.getId());
-    }
-
-    @Override
-    protected void onAbort() {
-        auditor.info(transformConfig.getId(), "Received abort request, stopping transform.");
-        logger.info("[{}] transform received abort request. Stopping indexer.", transformConfig.getId());
-        transformTask.shutdown();
-    }
-
-    @Override
-    protected void createCheckpoint(ActionListener<TransformCheckpoint> listener) {
-        checkpointProvider.createNextCheckpoint(getLastCheckpoint(), ActionListener.wrap(
-                checkpoint -> transformsConfigManager.putTransformCheckpoint(checkpoint,
-                    ActionListener.wrap(
-                        putCheckPointResponse -> listener.onResponse(checkpoint),
-                        createCheckpointException -> {
-                            logger.warn(new ParameterizedMessage("[{}] failed to create checkpoint.", getJobId()),
-                                createCheckpointException);
-                            listener.onFailure(
-                                new RuntimeException("Failed to create checkpoint due to " + createCheckpointException.getMessage(),
-                                    createCheckpointException));
-                        }
-                )),
-                getCheckPointException -> {
-                    logger.warn(new ParameterizedMessage("[{}] failed to retrieve checkpoint.", getJobId()),
-                        getCheckPointException);
-                    listener.onFailure(
-                        new RuntimeException("Failed to retrieve checkpoint due to " + getCheckPointException.getMessage(),
-                            getCheckPointException));
-                }
-        ));
-    }
-
-    @Override
-    protected void sourceHasChanged(ActionListener<Boolean> hasChangedListener) {
-        checkpointProvider.sourceHasChanged(getLastCheckpoint(),
-            ActionListener.wrap(
-                hasChanged -> {
-                    logger.trace("[{}] change detected [{}].", getJobId(), hasChanged);
-                    hasChangedListener.onResponse(hasChanged);
-                },
-                e -> {
-                    logger.warn(
-                        new ParameterizedMessage(
-                            "[{}] failed to detect changes for transform. Skipping update till next check.",
-                            getJobId()),
-                        e);
-                    auditor.warning(getJobId(),
-                        "Failed to detect changes for transform, skipping update till next check. Exception: "
-                            + e.getMessage());
-                    hasChangedListener.onResponse(false);
-                }));
-    }
-
-    private boolean isIrrecoverableFailure(Exception e) {
-        return e instanceof IndexNotFoundException
-            || e instanceof AggregationResultUtils.AggregationExtractionException
-            || e instanceof TransformConfigReloadingException;
-    }
-
-    synchronized void handleFailure(Exception e) {
-        logger.warn(new ParameterizedMessage("[{}] transform encountered an exception: ",
-            getJobId()),
-            e);
-        if (handleCircuitBreakingException(e)) {
-            return;
-        }
-
-        if (isIrrecoverableFailure(e) || failureCount.incrementAndGet() > transformTask.getNumFailureRetries()) {
-            String failureMessage = isIrrecoverableFailure(e) ?
-                "task encountered irrecoverable failure: " + e.getMessage() :
-                "task encountered more than " + transformTask.getNumFailureRetries() + " failures; latest failure: " + e.getMessage();
-            failIndexer(failureMessage);
-        } else {
-            // Since our schedule fires again very quickly after failures it is possible to run into the same failure numerous
-            // times in a row, very quickly. We do not want to spam the audit log with repeated failures, so only record the first one
-            if (e.getMessage().equals(lastAuditedExceptionMessage) == false) {
-                auditor.warning(getJobId(),
-                    "Transform encountered an exception: " + e.getMessage() +
-                        " Will attempt again at next scheduled trigger.");
-                lastAuditedExceptionMessage = e.getMessage();
-            }
-        }
-    }
-
-    @Override
-    protected void failIndexer(String failureMessage) {
-        logger.error("[{}] transform has failed; experienced: [{}].", getJobId(), failureMessage);
-        auditor.error(getJobId(), failureMessage);
-        transformTask.markAsFailed(failureMessage, ActionListener.wrap(
-            r -> {
-                // Successfully marked as failed, reset counter so that task can be restarted
-                failureCount.set(0);
-            }, e -> {}));
     }
 
     // Considered a recoverable indexing failure
@@ -666,9 +332,4 @@ class ClientTransformIndexer extends TransformIndexer {
         }
     }
 
-    private static class TransformConfigReloadingException extends ElasticsearchException {
-        TransformConfigReloadingException(String msg, Throwable cause, Object... args) {
-            super(msg, cause, args);
-        }
-    }
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java

@@ -8,17 +8,19 @@ package org.elasticsearch.xpack.transform.transforms;
 
 import org.elasticsearch.client.Client;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformProgress;
 import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider;
 import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
+import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex;
 import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
 
 import java.util.Map;
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 
 class ClientTransformIndexerBuilder {
@@ -34,29 +36,35 @@ class ClientTransformIndexerBuilder {
     private TransformProgress progress;
     private TransformCheckpoint lastCheckpoint;
     private TransformCheckpoint nextCheckpoint;
+    private SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex;
     private boolean shouldStopAtCheckpoint;
 
     ClientTransformIndexerBuilder() {
         this.initialStats = new TransformIndexerStats();
     }
 
-    ClientTransformIndexer build(TransformTask parentTask) {
+    ClientTransformIndexer build(Executor executor, TransformContext context) {
         CheckpointProvider checkpointProvider = transformsCheckpointService.getCheckpointProvider(transformConfig);
 
-        return new ClientTransformIndexer(this.transformsConfigManager,
+        return new ClientTransformIndexer(
+            executor,
+            transformsConfigManager,
             checkpointProvider,
+            new TransformProgressGatherer(client),
             new AtomicReference<>(this.indexerState),
-            this.initialPosition,
+            initialPosition,
-            this.client,
+            client,
-            this.auditor,
+            auditor,
-            this.initialStats,
+            initialStats,
-            this.transformConfig,
+            transformConfig,
-            this.fieldMappings,
+            fieldMappings,
-            this.progress,
+            progress,
-            this.lastCheckpoint,
+            TransformCheckpoint.isNullOrEmpty(lastCheckpoint) ? TransformCheckpoint.EMPTY : lastCheckpoint,
-            this.nextCheckpoint,
+            TransformCheckpoint.isNullOrEmpty(nextCheckpoint) ? TransformCheckpoint.EMPTY : nextCheckpoint,
-            parentTask,
+            seqNoPrimaryTermAndIndex,
-            this.shouldStopAtCheckpoint);
+            context,
+            shouldStopAtCheckpoint
+        );
     }
 
     ClientTransformIndexerBuilder setShouldStopAtCheckpoint(boolean shouldStopAtCheckpoint) {
@@ -127,4 +135,10 @@ class ClientTransformIndexerBuilder {
         this.nextCheckpoint = nextCheckpoint;
         return this;
     }
+
+    ClientTransformIndexerBuilder setSeqNoPrimaryTermAndIndex(SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex) {
+        this.seqNoPrimaryTermAndIndex = seqNoPrimaryTermAndIndex;
+        return this;
+    }
+
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.transform.transforms;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
+import org.elasticsearch.xpack.transform.Transform;
+
+import java.time.Instant;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+class TransformContext {
+
+    public interface Listener {
+        void shutdown();
+
+        void fail(String failureMessage, ActionListener<Void> listener);
+    }
+
+    private final AtomicReference<TransformTaskState> taskState;
+    private final AtomicReference<String> stateReason;
+    private final Listener taskListener;
+    private volatile int numFailureRetries = Transform.DEFAULT_FAILURE_RETRIES;
+    private final AtomicInteger failureCount;
+    private volatile Instant changesLastDetectedAt;
+    private volatile boolean shouldStopAtCheckpoint;
+
+    // the checkpoint of this transform, storing the checkpoint until data indexing from source to dest is _complete_
+    // Note: Each indexer run creates a new future checkpoint which becomes the current checkpoint only after the indexer run finished
+    private final AtomicLong currentCheckpoint;
+
+    TransformContext(final TransformTaskState taskState, String stateReason, long currentCheckpoint, Listener taskListener) {
+        this.taskState = new AtomicReference<>(taskState);
+        this.stateReason = new AtomicReference<>(stateReason);
+        this.currentCheckpoint = new AtomicLong(currentCheckpoint);
+        this.taskListener = taskListener;
+        this.failureCount = new AtomicInteger(0);
+        this.shouldStopAtCheckpoint = shouldStopAtCheckpoint;
+    }
+
+    TransformTaskState getTaskState() {
+        return taskState.get();
+    }
+
+    void setTaskState(TransformTaskState newState) {
+        taskState.set(newState);
+    }
+
+    boolean setTaskState(TransformTaskState oldState, TransformTaskState newState) {
+        return taskState.compareAndSet(oldState, newState);
+    }
+
+    void resetTaskState() {
+        taskState.set(TransformTaskState.STARTED);
+        stateReason.set(null);
+    }
+
+    void setTaskStateToFailed(String reason) {
+        taskState.set(TransformTaskState.FAILED);
+        stateReason.set(reason);
+    }
+
+    void resetReasonAndFailureCounter() {
+        stateReason.set(null);
+        failureCount.set(0);
+    }
+
+    String getStateReason() {
+        return stateReason.get();
+    }
+
+    void setCheckpoint(long newValue) {
+        currentCheckpoint.set(newValue);
+    }
+
+    long getCheckpoint() {
+        return currentCheckpoint.get();
+    }
+
+    long getAndIncrementCheckpoint() {
+        return currentCheckpoint.getAndIncrement();
+    }
+
+    void setNumFailureRetries(int numFailureRetries) {
+        this.numFailureRetries = numFailureRetries;
+    }
+
+    int getNumFailureRetries() {
+        return numFailureRetries;
+    }
+
+    int getAndIncrementFailureCount() {
+        return failureCount.getAndIncrement();
+    }
+
+    void setChangesLastDetectedAt(Instant time) {
+        changesLastDetectedAt = time;
+    }
+
+    Instant getChangesLastDetectedAt() {
+        return changesLastDetectedAt;
+    }
+
+    public boolean shouldStopAtCheckpoint() {
+        return shouldStopAtCheckpoint;
+    }
+
+    public void setShouldStopAtCheckpoint(boolean shouldStopAtCheckpoint) {
+        this.shouldStopAtCheckpoint = shouldStopAtCheckpoint;
+    }
+
+    void shutdown() {
+        taskListener.shutdown();
+    }
+
+    void markAsFailed(String failureMessage) {
+        taskListener
+            .fail(
+                failureMessage,
+                ActionListener
+                    .wrap(
+                        r -> {
+                            // Successfully marked as failed, reset counter so that task can be restarted
+                            failureCount.set(0);
+                        },
+                        e -> {}
+                    )
+            );
+    }
+
+}

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java

@@ -8,6 +8,9 @@ package org.elasticsearch.xpack.transform.transforms;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
@@ -18,6 +21,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.Aggregations;
@@ -29,20 +33,24 @@ import org.elasticsearch.xpack.core.indexing.IndexerState;
 import org.elasticsearch.xpack.core.indexing.IterationResult;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformProgress;
+import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
 import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
+import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
+import org.elasticsearch.xpack.transform.transforms.pivot.AggregationResultUtils;
 import org.elasticsearch.xpack.transform.transforms.pivot.Pivot;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
@@ -73,24 +81,41 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
     public static final String COMPOSITE_AGGREGATION_NAME = "_transform";
     private static final Logger logger = LogManager.getLogger(TransformIndexer.class);
 
+    protected final TransformConfigManager transformsConfigManager;
+    private final CheckpointProvider checkpointProvider;
+    private final TransformProgressGatherer progressGatherer;
+
     protected final TransformAuditor auditor;
+    protected final TransformContext context;
 
     protected volatile TransformConfig transformConfig;
-    protected volatile TransformProgress progress;
+    private volatile TransformProgress progress;
+    protected volatile boolean auditBulkFailures = true;
+    // Indicates that the source has changed for the current run
+    protected volatile boolean hasSourceChanged = true;
+
     private final Map<String, String> fieldMappings;
 
     private Pivot pivot;
     private int pageSize = 0;
-    protected volatile TransformCheckpoint lastCheckpoint;
+    private long logEvery = 1;
-    protected volatile TransformCheckpoint nextCheckpoint;
+    private long logCount = 0;
+    private volatile TransformCheckpoint lastCheckpoint;
+    private volatile TransformCheckpoint nextCheckpoint;
 
+    // Keeps track of the last exception that was written to our audit, keeps us from spamming the audit index
+    private volatile String lastAuditedExceptionMessage = null;
     private volatile RunState runState;
 
     // hold information for continuous mode (partial updates)
     private volatile Map<String, Set<String>> changedBuckets;
     private volatile Map<String, Object> changedBucketsAfterKey;
 
-    public TransformIndexer(Executor executor,
+    public TransformIndexer(
+        Executor executor,
+        TransformConfigManager transformsConfigManager,
+        CheckpointProvider checkpointProvider,
+        TransformProgressGatherer progressGatherer,
         TransformAuditor auditor,
         TransformConfig transformConfig,
         Map<String, String> fieldMappings,
@@ -99,20 +124,25 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         TransformIndexerStats jobStats,
         TransformProgress transformProgress,
         TransformCheckpoint lastCheckpoint,
-                            TransformCheckpoint nextCheckpoint) {
+        TransformCheckpoint nextCheckpoint,
+        TransformContext context
+    ) {
         super(executor, initialState, initialPosition, jobStats);
-        this.auditor = Objects.requireNonNull(auditor);
+        this.transformsConfigManager = ExceptionsHelper.requireNonNull(transformsConfigManager, "transformsConfigManager");
+        this.checkpointProvider = ExceptionsHelper.requireNonNull(checkpointProvider, "checkpointProvider");
+        this.progressGatherer = ExceptionsHelper.requireNonNull(progressGatherer, "progressGatherer");
+        this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor");
         this.transformConfig = ExceptionsHelper.requireNonNull(transformConfig, "transformConfig");
         this.fieldMappings = ExceptionsHelper.requireNonNull(fieldMappings, "fieldMappings");
         this.progress = transformProgress;
-        this.lastCheckpoint = lastCheckpoint;
+        this.lastCheckpoint = ExceptionsHelper.requireNonNull(lastCheckpoint, "lastCheckpoint");
-        this.nextCheckpoint = nextCheckpoint;
+        this.nextCheckpoint = ExceptionsHelper.requireNonNull(nextCheckpoint, "nextCheckpoint");
+        this.context = ExceptionsHelper.requireNonNull(context, "context");
+
         // give runState a default
         this.runState = RunState.FULL_RUN;
     }
 
-    protected abstract void failIndexer(String message);
-
     public int getPageSize() {
         return pageSize;
     }
@@ -146,13 +176,61 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         return nextCheckpoint;
     }
 
+    public CheckpointProvider getCheckpointProvider() {
+        return checkpointProvider;
+    }
+
     /**
      * Request a checkpoint
      */
-    protected abstract void createCheckpoint(ActionListener<TransformCheckpoint> listener);
+    protected void createCheckpoint(ActionListener<TransformCheckpoint> listener) {
+        checkpointProvider
+            .createNextCheckpoint(
+                getLastCheckpoint(),
+                ActionListener
+                    .wrap(
+                        checkpoint -> transformsConfigManager
+                            .putTransformCheckpoint(
+                                checkpoint,
+                                ActionListener.wrap(putCheckPointResponse -> listener.onResponse(checkpoint), createCheckpointException -> {
+                                    logger
+                                        .warn(
+                                            new ParameterizedMessage("[{}] failed to create checkpoint.", getJobId()),
+                                            createCheckpointException
+                                        );
+                                    listener
+                                        .onFailure(
+                                            new RuntimeException(
+                                                "Failed to create checkpoint due to " + createCheckpointException.getMessage(),
+                                                createCheckpointException
+                                            )
+                                        );
+                                })
+                            ),
+                        getCheckPointException -> {
+                            logger
+                                .warn(new ParameterizedMessage("[{}] failed to retrieve checkpoint.", getJobId()), getCheckPointException);
+                            listener
+                                .onFailure(
+                                    new RuntimeException(
+                                        "Failed to retrieve checkpoint due to " + getCheckPointException.getMessage(),
+                                        getCheckPointException
+                                    )
+                                );
+                        }
+                    )
+            );
+    }
 
     @Override
     protected void onStart(long now, ActionListener<Boolean> listener) {
+        if (context.getTaskState() == TransformTaskState.FAILED) {
+            logger.debug("[{}] attempted to start while failed.", getJobId());
+            listener.onFailure(new ElasticsearchException("Attempted to start a failed transform [{}].", getJobId()));
+            return;
+        }
+
+        ActionListener<Void> finalListener = ActionListener.wrap(r -> {
             try {
                 pivot = new Pivot(getConfig().getPivotConfig());
 
@@ -165,6 +243,86 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
                 listener.onResponse(true);
             } catch (Exception e) {
                 listener.onFailure(e);
+                return;
+            }
+        }, listener::onFailure);
+
+        // On each run, we need to get the total number of docs and reset the count of processed docs
+        // Since multiple checkpoints can be executed in the task while it is running on the same node, we need to gather
+        // the progress here, and not in the executor.
+        ActionListener<Void> updateConfigListener = ActionListener.wrap(updateConfigResponse -> {
+            if (initialRun()) {
+                createCheckpoint(ActionListener.wrap(cp -> {
+                    nextCheckpoint = cp;
+                    // If nextCheckpoint > 1, this means that we are now on the checkpoint AFTER the batch checkpoint
+                    // Consequently, the idea of percent complete no longer makes sense.
+                    if (nextCheckpoint.getCheckpoint() > 1) {
+                        progress = new TransformProgress(null, 0L, 0L);
+                        finalListener.onResponse(null);
+                        return;
+                    }
+                    progressGatherer.getInitialProgress(buildFilterQuery(), getConfig(), ActionListener.wrap(newProgress -> {
+                        logger.trace("[{}] reset the progress from [{}] to [{}].", getJobId(), progress, newProgress);
+                        progress = newProgress;
+                        finalListener.onResponse(null);
+                    }, failure -> {
+                        progress = null;
+                        logger.warn(new ParameterizedMessage("[{}] unable to load progress information for task.", getJobId()), failure);
+                        finalListener.onResponse(null);
+                    }));
+                }, listener::onFailure));
+            } else {
+                finalListener.onResponse(null);
+            }
+        }, listener::onFailure);
+
+        // If we are continuous, we will want to verify we have the latest stored configuration
+        ActionListener<Void> changedSourceListener = ActionListener.wrap(r -> {
+            if (isContinuous()) {
+                transformsConfigManager.getTransformConfiguration(getJobId(), ActionListener.wrap(config -> {
+                    transformConfig = config;
+                    logger.debug("[{}] successfully refreshed transform config from index.", getJobId());
+                    updateConfigListener.onResponse(null);
+                }, failure -> {
+                    String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_RELOAD_TRANSFORM_CONFIGURATION, getJobId());
+                    logger.error(msg, failure);
+                    // If the transform config index or the transform config is gone, something serious occurred
+                    // We are in an unknown state and should fail out
+                    if (failure instanceof ResourceNotFoundException) {
+                        updateConfigListener.onFailure(new TransformConfigReloadingException(msg, failure));
+                    } else {
+                        auditor.warning(getJobId(), msg);
+                        updateConfigListener.onResponse(null);
+                    }
+                }));
+            } else {
+                updateConfigListener.onResponse(null);
+            }
+        }, listener::onFailure);
+
+        // If we are not on the initial batch checkpoint and its the first pass of whatever continuous checkpoint we are on,
+        // we should verify if there are local changes based on the sync config. If not, do not proceed further and exit.
+        if (context.getCheckpoint() > 0 && initialRun()) {
+            sourceHasChanged(ActionListener.wrap(hasChanged -> {
+                hasSourceChanged = hasChanged;
+                if (hasChanged) {
+                    context.setChangesLastDetectedAt(Instant.now());
+                    logger.debug("[{}] source has changed, triggering new indexer run.", getJobId());
+                    changedSourceListener.onResponse(null);
+                } else {
+                    logger.trace("[{}] source has not changed, finish indexer early.", getJobId());
+                    // No changes, stop executing
+                    listener.onResponse(false);
+                }
+            }, failure -> {
+                // If we failed determining if the source changed, it's safer to assume there were changes.
+                // We should allow the failure path to complete as normal
+                hasSourceChanged = true;
+                listener.onFailure(failure);
+            }));
+        } else {
+            hasSourceChanged = true;
+            changedSourceListener.onResponse(null);
         }
     }
 
@@ -174,10 +332,63 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
 
     @Override
     protected void onFinish(ActionListener<Void> listener) {
+        try {
+            // This indicates an early exit since no changes were found.
+            // So, don't treat this like a checkpoint being completed, as no work was done.
+            if (hasSourceChanged == false) {
+                if (context.shouldStopAtCheckpoint()) {
+                    stop();
+                }
+                listener.onResponse(null);
+                return;
+            }
+
             // reset the page size, so we do not memorize a low page size forever
             pageSize = pivot.getInitialPageSize();
             // reset the changed bucket to free memory
             changedBuckets = null;
+
+            long checkpoint = context.getAndIncrementCheckpoint();
+            lastCheckpoint = getNextCheckpoint();
+            nextCheckpoint = null;
+            // Reset our failure count as we have finished and may start again with a new checkpoint
+            context.resetReasonAndFailureCounter();
+
+            // With bucket_selector we could have read all the buckets and completed the transform
+            // but not "see" all the buckets since they were filtered out. Consequently, progress would
+            // show less than 100% even though we are done.
+            // NOTE: this method is called in the same thread as the processing thread.
+            // Theoretically, there should not be a race condition with updating progress here.
+            // NOTE 2: getPercentComplete should only NOT be null on the first (batch) checkpoint
+            if (progress != null && progress.getPercentComplete() != null && progress.getPercentComplete() < 100.0) {
+                progress.incrementDocsProcessed(progress.getTotalDocs() - progress.getDocumentsProcessed());
+            }
+            // If the last checkpoint is now greater than 1, that means that we have just processed the first
+            // continuous checkpoint and should start recording the exponential averages
+            if (lastCheckpoint != null && lastCheckpoint.getCheckpoint() > 1) {
+                long docsIndexed = 0;
+                long docsProcessed = 0;
+                // This should not happen as we simply create a new one when we reach continuous checkpoints
+                // but this is a paranoid `null` check
+                if (progress != null) {
+                    docsIndexed = progress.getDocumentsIndexed();
+                    docsProcessed = progress.getDocumentsProcessed();
+                }
+                long durationMs = System.currentTimeMillis() - lastCheckpoint.getTimestamp();
+                getStats().incrementCheckpointExponentialAverages(durationMs < 0 ? 0 : durationMs, docsIndexed, docsProcessed);
+            }
+            if (shouldAuditOnFinish(checkpoint)) {
+                auditor.info(getJobId(), "Finished indexing for transform checkpoint [" + checkpoint + "].");
+            }
+            logger.debug("[{}] finished indexing for transform checkpoint [{}].", getJobId(), checkpoint);
+            auditBulkFailures = true;
+            if (context.shouldStopAtCheckpoint()) {
+                stop();
+            }
+            listener.onResponse(null);
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
     }
 
     @Override
@@ -186,13 +397,16 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         // Treat this as a "we reached the end".
         // This should only happen when all underlying indices have gone away. Consequently, there is no more data to read.
         if (aggregations == null) {
-            logger.info("[" + getJobId() + "] unexpected null aggregations in search response. " +
+            logger
-                "Source indices have been deleted or closed.");
+                .info("[{}] unexpected null aggregations in search response. " + "Source indices have been deleted or closed.", getJobId());
-            auditor.info(getJobId(),
+            auditor
-                "Source indices have been deleted or closed. " +
+                .info(
-                    "Please verify that these indices exist and are open [" +
+                    getJobId(),
-                    Strings.arrayToCommaDelimitedString(getConfig().getSource().getIndex()) +
+                    "Source indices have been deleted or closed. "
-                    "].");
+                        + "Please verify that these indices exist and are open ["
+                        + Strings.arrayToCommaDelimitedString(getConfig().getSource().getIndex())
+                        + "]."
+                );
             return new IterationResult<>(Collections.emptyList(), null, true);
         }
         final CompositeAggregation agg = aggregations.get(COMPOSITE_AGGREGATION_NAME);
@@ -207,11 +421,101 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
 
             default:
                 // Any other state is a bug, should not happen
-            logger.warn("Encountered unexpected run state [" + runState + "]");
+                logger.warn("[{}] Encountered unexpected run state [{}]", getJobId(), runState);
                 throw new IllegalStateException("DataFrame indexer job encountered an illegal state [" + runState + "]");
         }
     }
 
+    @Override
+    public synchronized boolean maybeTriggerAsyncJob(long now) {
+        if (context.getTaskState() == TransformTaskState.FAILED) {
+            logger.debug("[{}] schedule was triggered for transform but task is failed. Ignoring trigger.", getJobId());
+            return false;
+        }
+
+        // ignore trigger if indexer is running, prevents log spam in A2P indexer
+        IndexerState indexerState = getState();
+        if (IndexerState.INDEXING.equals(indexerState) || IndexerState.STOPPING.equals(indexerState)) {
+            logger.debug("[{}] indexer for transform has state [{}]. Ignoring trigger.", getJobId(), indexerState);
+            return false;
+        }
+
+        return super.maybeTriggerAsyncJob(now);
+    }
+
+    @Override
+    protected void onFailure(Exception exc) {
+        // the failure handler must not throw an exception due to internal problems
+        try {
+            handleFailure(exc);
+        } catch (Exception e) {
+            logger.error(new ParameterizedMessage("[{}] transform encountered an unexpected internal exception: ", getJobId()), e);
+        }
+    }
+
+    @Override
+    protected void onStop() {
+        auditor.info(transformConfig.getId(), "Transform has stopped.");
+        logger.info("[{}] transform has stopped.", transformConfig.getId());
+    }
+
+    @Override
+    protected void onAbort() {
+        auditor.info(transformConfig.getId(), "Received abort request, stopping transform.");
+        logger.info("[{}] transform received abort request. Stopping indexer.", transformConfig.getId());
+        context.shutdown();
+    }
+
+    synchronized void handleFailure(Exception e) {
+        logger.warn(new ParameterizedMessage("[{}] transform encountered an exception: ", getJobId()), e);
+        if (handleCircuitBreakingException(e)) {
+            return;
+        }
+
+        if (isIrrecoverableFailure(e) || context.getAndIncrementFailureCount() > context.getNumFailureRetries()) {
+            String failureMessage = isIrrecoverableFailure(e)
+                ? "task encountered irrecoverable failure: " + e.getMessage()
+                : "task encountered more than " + context.getNumFailureRetries() + " failures; latest failure: " + e.getMessage();
+            failIndexer(failureMessage);
+        } else {
+            // Since our schedule fires again very quickly after failures it is possible to run into the same failure numerous
+            // times in a row, very quickly. We do not want to spam the audit log with repeated failures, so only record the first one
+            if (e.getMessage().equals(lastAuditedExceptionMessage) == false) {
+                auditor
+                    .warning(
+                        getJobId(),
+                        "Transform encountered an exception: " + e.getMessage() + " Will attempt again at next scheduled trigger."
+                    );
+                lastAuditedExceptionMessage = e.getMessage();
+            }
+        }
+    }
+
+    private void sourceHasChanged(ActionListener<Boolean> hasChangedListener) {
+        checkpointProvider.sourceHasChanged(getLastCheckpoint(), ActionListener.wrap(hasChanged -> {
+            logger.trace("[{}] change detected [{}].", getJobId(), hasChanged);
+            hasChangedListener.onResponse(hasChanged);
+        }, e -> {
+            logger
+                .warn(
+                    new ParameterizedMessage("[{}] failed to detect changes for transform. Skipping update till next check.", getJobId()),
+                    e
+                );
+            auditor
+                .warning(
+                    getJobId(),
+                    "Failed to detect changes for transform, skipping update till next check. Exception: " + e.getMessage()
+                );
+            hasChangedListener.onResponse(false);
+        }));
+    }
+
+    private boolean isIrrecoverableFailure(Exception e) {
+        return e instanceof IndexNotFoundException
+            || e instanceof AggregationResultUtils.AggregationExtractionException
+            || e instanceof TransformConfigReloadingException;
+    }
+
     private IterationResult<TransformIndexerPosition> processBuckets(final CompositeAggregation agg) {
         // we reached the end
         if (agg.getBuckets().isEmpty()) {
@@ -221,13 +525,16 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         long docsBeforeProcess = getStats().getNumDocuments();
 
         TransformIndexerPosition oldPosition = getPosition();
-        TransformIndexerPosition newPosition = new TransformIndexerPosition(agg.afterKey(),
+        TransformIndexerPosition newPosition = new TransformIndexerPosition(
-                oldPosition != null ? getPosition().getBucketsPosition() : null);
+            agg.afterKey(),
+            oldPosition != null ? getPosition().getBucketsPosition() : null
+        );
 
         IterationResult<TransformIndexerPosition> result = new IterationResult<>(
             processBucketsToIndexRequests(agg).collect(Collectors.toList()),
             newPosition,
-                agg.getBuckets().isEmpty());
+            agg.getBuckets().isEmpty()
+        );
 
         // NOTE: progress is also mutated in ClientDataFrameIndexer#onFinished
         if (progress != null) {
@@ -247,14 +554,12 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
             // reset the runState to fetch changed buckets
             runState = RunState.PARTIAL_RUN_IDENTIFY_CHANGES;
             // advance the cursor for changed bucket detection
-            return new IterationResult<>(Collections.emptyList(),
+            return new IterationResult<>(Collections.emptyList(), new TransformIndexerPosition(null, changedBucketsAfterKey), false);
-                    new TransformIndexerPosition(null, changedBucketsAfterKey), false);
         }
 
         return processBuckets(agg);
     }
 
-
     private IterationResult<TransformIndexerPosition> processChangedBuckets(final CompositeAggregation agg) {
         // initialize the map of changed buckets, the map might be empty if source do not require/implement
         // changed bucket detection
@@ -270,11 +575,7 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         // else
 
         // collect all buckets that require the update
-        agg.getBuckets().stream().forEach(bucket -> {
+        agg.getBuckets().stream().forEach(bucket -> { bucket.getKey().forEach((k, v) -> { changedBuckets.get(k).add(v.toString()); }); });
-            bucket.getKey().forEach((k, v) -> {
-                changedBuckets.get(k).add(v.toString());
-            });
-        });
 
         // remember the after key but do not store it in the state yet (in the failure we need to retrieve it again)
         changedBucketsAfterKey = agg.afterKey();
@@ -335,8 +636,7 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         TransformConfig config = getConfig();
         if (this.isContinuous()) {
 
-            BoolQueryBuilder filteredQuery = new BoolQueryBuilder()
+            BoolQueryBuilder filteredQuery = new BoolQueryBuilder().filter(pivotQueryBuilder);
-                .filter(pivotQueryBuilder);
 
             if (lastCheckpoint != null) {
                 filteredQuery.filter(config.getSyncConfig().getRangeQuery(lastCheckpoint, nextCheckpoint));
@@ -356,8 +656,7 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         SearchRequest searchRequest = new SearchRequest(getConfig().getSource().getIndex())
             .allowPartialSearchResults(false)
             .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().size(0);
-                .size(0);
 
         switch (runState) {
             case FULL_RUN:
@@ -389,8 +688,7 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         if (isContinuous()) {
             BoolQueryBuilder filteredQuery = new BoolQueryBuilder()
                 .filter(pivotQueryBuilder)
-                    .filter(config.getSyncConfig()
+                .filter(config.getSyncConfig().getRangeQuery(nextCheckpoint));
-                            .getRangeQuery(nextCheckpoint));
             sourceBuilder.query(filteredQuery);
         } else {
             sourceBuilder.query(pivotQueryBuilder);
@@ -413,9 +711,9 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         QueryBuilder pivotQueryBuilder = getConfig().getSource().getQueryConfig().getQuery();
 
         TransformConfig config = getConfig();
-        BoolQueryBuilder filteredQuery = new BoolQueryBuilder().
+        BoolQueryBuilder filteredQuery = new BoolQueryBuilder()
-                filter(pivotQueryBuilder).
+            .filter(pivotQueryBuilder)
-                filter(config.getSyncConfig().getRangeQuery(lastCheckpoint, nextCheckpoint));
+            .filter(config.getSyncConfig().getRangeQuery(lastCheckpoint, nextCheckpoint));
 
         sourceBuilder.query(filteredQuery);
 
@@ -435,8 +733,7 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
 
         BoolQueryBuilder filteredQuery = new BoolQueryBuilder()
             .filter(pivotQueryBuilder)
-                .filter(config.getSyncConfig()
+            .filter(config.getSyncConfig().getRangeQuery(nextCheckpoint));
-                        .getRangeQuery(nextCheckpoint));
 
         if (changedBuckets != null && changedBuckets.isEmpty() == false) {
             QueryBuilder pivotFilter = pivot.filterBuckets(changedBuckets);
@@ -469,8 +766,11 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
             return false;
         }
 
-        double reducingFactor = Math.min((double) circuitBreakingException.getByteLimit() / circuitBreakingException.getBytesWanted(),
+        double reducingFactor = Math
-                1 - (Math.log10(pageSize) * 0.1));
+            .min(
+                (double) circuitBreakingException.getByteLimit() / circuitBreakingException.getBytesWanted(),
+                1 - (Math.log10(pageSize) * 0.1)
+            );
 
         int newPageSize = (int) Math.round(reducingFactor * pageSize);
 
@@ -480,8 +780,7 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
             return true;
         }
 
-        String message = TransformMessages.getMessage(TransformMessages.LOG_TRANSFORM_PIVOT_REDUCE_PAGE_SIZE, pageSize,
+        String message = TransformMessages.getMessage(TransformMessages.LOG_TRANSFORM_PIVOT_REDUCE_PAGE_SIZE, pageSize, newPageSize);
-                newPageSize);
         auditor.info(getJobId(), message);
         logger.info("Data frame transform [" + getJobId() + "]:" + message);
 
@@ -489,6 +788,35 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         return true;
     }
 
+    protected void failIndexer(String failureMessage) {
+        logger.error("[{}] transform has failed; experienced: [{}].", getJobId(), failureMessage);
+        auditor.error(getJobId(), failureMessage);
+        context.markAsFailed(failureMessage);
+    }
+
+    /**
+     * Indicates if an audit message should be written when onFinish is called for the given checkpoint
+     * We audit the first checkpoint, and then every 10 checkpoints until completedCheckpoint == 99
+     * Then we audit every 100, until completedCheckpoint == 999
+     *
+     * Then we always audit every 1_000 checkpoints
+     *
+     * @param completedCheckpoint The checkpoint that was just completed
+     * @return {@code true} if an audit message should be written
+     */
+    protected boolean shouldAuditOnFinish(long completedCheckpoint) {
+        if (++logCount % logEvery != 0) {
+            return false;
+        }
+        if (completedCheckpoint == 0) {
+            return true;
+        }
+        int log10Checkpoint = (int) Math.floor(Math.log10(completedCheckpoint));
+        logEvery = log10Checkpoint >= 3 ? 1_000 : (int) Math.pow(10.0, log10Checkpoint);
+        logCount = 0;
+        return true;
+    }
+
     private RunState determineRunStateAtStart() {
         // either 1st run or not a continuous data frame
         if (nextCheckpoint.getCheckpoint() == 1 || isContinuous() == false) {
@@ -530,5 +858,9 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
         return null;
     }
 
-    protected abstract void sourceHasChanged(ActionListener<Boolean> hasChangedListener);
+    static class TransformConfigReloadingException extends ElasticsearchException {
+        TransformConfigReloadingException(String msg, Throwable cause, Object... args) {
+            super(msg, cause, args);
+        }
+    }
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java

@@ -69,14 +69,16 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
     private final TransformAuditor auditor;
     private volatile int numFailureRetries;
 
-    public TransformPersistentTasksExecutor(Client client,
+    public TransformPersistentTasksExecutor(
+        Client client,
         TransformConfigManager transformsConfigManager,
         TransformCheckpointService transformsCheckpointService,
         SchedulerEngine schedulerEngine,
         TransformAuditor auditor,
         ThreadPool threadPool,
         ClusterService clusterService,
-                                            Settings settings) {
+        Settings settings
+    ) {
         super(TransformField.TASK_NAME, Transform.TASK_THREAD_POOL_NAME);
         this.client = client;
         this.transformsConfigManager = transformsConfigManager;
@@ -85,43 +87,50 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
         this.auditor = auditor;
         this.threadPool = threadPool;
         this.clusterService = clusterService;
-        this.numFailureRetries = TransformTask.NUM_FAILURE_RETRIES_SETTING.get(settings);
+        this.numFailureRetries = Transform.NUM_FAILURE_RETRIES_SETTING.get(settings);
-        clusterService.getClusterSettings()
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(Transform.NUM_FAILURE_RETRIES_SETTING, this::setNumFailureRetries);
-            .addSettingsUpdateConsumer(TransformTask.NUM_FAILURE_RETRIES_SETTING, this::setNumFailureRetries);
     }
 
     @Override
     public PersistentTasksCustomMetaData.Assignment getAssignment(TransformTaskParams params, ClusterState clusterState) {
         List<String> unavailableIndices = verifyIndicesPrimaryShardsAreActive(clusterState);
         if (unavailableIndices.size() != 0) {
-            String reason = "Not starting transform [" + params.getId() + "], " +
+            String reason = "Not starting transform ["
-                "because not all primary shards are active for the following indices [" +
+                + params.getId()
-                String.join(",", unavailableIndices) + "]";
+                + "], "
+                + "because not all primary shards are active for the following indices ["
+                + String.join(",", unavailableIndices)
+                + "]";
             logger.debug(reason);
             return new PersistentTasksCustomMetaData.Assignment(null, reason);
         }
 
         // see gh#48019 disable assignment if any node is using 7.2 or 7.3
         if (clusterState.getNodes().getMinNodeVersion().before(Version.V_7_4_0)) {
-            String reason = "Not starting transform [" + params.getId() + "], " +
+            String reason = "Not starting transform ["
-                "because cluster contains nodes with version older than 7.4.0";
+                + params.getId()
+                + "], "
+                + "because cluster contains nodes with version older than 7.4.0";
             logger.debug(reason);
             return new PersistentTasksCustomMetaData.Assignment(null, reason);
         }
 
-        DiscoveryNode discoveryNode = selectLeastLoadedNode(clusterState, (node) ->
+        DiscoveryNode discoveryNode = selectLeastLoadedNode(
-            node.isDataNode() &&
+            clusterState,
-            node.getVersion().onOrAfter(params.getVersion())
+            (node) -> node.isDataNode() && node.getVersion().onOrAfter(params.getVersion())
         );
         return discoveryNode == null ? NO_NODE_FOUND : new PersistentTasksCustomMetaData.Assignment(discoveryNode.getId(), "");
     }
 
     static List<String> verifyIndicesPrimaryShardsAreActive(ClusterState clusterState) {
         IndexNameExpressionResolver resolver = new IndexNameExpressionResolver();
-        String[] indices = resolver.concreteIndexNames(clusterState,
+        String[] indices = resolver
+            .concreteIndexNames(
+                clusterState,
                 IndicesOptions.lenientExpandOpen(),
                 TransformInternalIndexConstants.INDEX_NAME_PATTERN,
-            TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED);
+                TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED
+            );
         List<String> unavailableIndices = new ArrayList<>(indices.length);
         for (String index : indices) {
             IndexRoutingTable routingTable = clusterState.getRoutingTable().index(index);
@@ -145,8 +154,7 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
         // We want the rest of the state to be populated in the task when it is loaded on the node so that users can force start it again
         // later if they want.
 
-        final ClientTransformIndexerBuilder indexerBuilder =
+        final ClientTransformIndexerBuilder indexerBuilder = new ClientTransformIndexerBuilder()
-            new ClientTransformIndexerBuilder()
             .setAuditor(auditor)
             .setClient(client)
             .setTransformsCheckpointService(transformCheckpointService)
@@ -154,18 +162,18 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
 
         final SetOnce<TransformState> stateHolder = new SetOnce<>();
 
-        ActionListener<StartTransformAction.Response> startTaskListener = ActionListener.wrap(
+        ActionListener<StartTransformAction.Response> startTaskListener = ActionListener
-            response -> logger.info("[{}] successfully completed and scheduled task in node operation", transformId),
+            .wrap(response -> logger.info("[{}] successfully completed and scheduled task in node operation", transformId), failure -> {
-            failure -> {
+                auditor
-                auditor.error(transformId, "Failed to start transform. " +
+                    .error(
-                    "Please stop and attempt to start again. Failure: " + failure.getMessage());
+                        transformId,
-                logger.error("Failed to start task ["+ transformId +"] in node operation", failure);
+                        "Failed to start transform. " + "Please stop and attempt to start again. Failure: " + failure.getMessage()
-            }
                     );
+                logger.error("Failed to start task [" + transformId + "] in node operation", failure);
+            });
 
         // <7> load next checkpoint
-        ActionListener<TransformCheckpoint> getTransformNextCheckpointListener = ActionListener.wrap(
+        ActionListener<TransformCheckpoint> getTransformNextCheckpointListener = ActionListener.wrap(nextCheckpoint -> {
-                nextCheckpoint -> {
 
             if (nextCheckpoint.isEmpty()) {
                 // extra safety: reset position and progress if next checkpoint is empty
@@ -173,60 +181,57 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
                 indexerBuilder.setInitialPosition(null);
                 indexerBuilder.setProgress(null);
             } else {
-                        logger.trace("[{}] Loaded next checkpoint [{}] found, starting the task", transformId,
+                logger.trace("[{}] Loaded next checkpoint [{}] found, starting the task", transformId, nextCheckpoint.getCheckpoint());
-                                nextCheckpoint.getCheckpoint());
                 indexerBuilder.setNextCheckpoint(nextCheckpoint);
             }
 
             final long lastCheckpoint = stateHolder.get().getCheckpoint();
 
             startTask(buildTask, indexerBuilder, lastCheckpoint, startTaskListener);
-                },
+        }, error -> {
-                error -> {
             // TODO: do not use the same error message as for loading the last checkpoint
             String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_LOAD_TRANSFORM_CHECKPOINT, transformId);
             logger.error(msg, error);
             markAsFailed(buildTask, msg);
-                }
+        });
-        );
 
         // <6> load last checkpoint
-        ActionListener<TransformCheckpoint> getTransformLastCheckpointListener = ActionListener.wrap(
+        ActionListener<TransformCheckpoint> getTransformLastCheckpointListener = ActionListener.wrap(lastCheckpoint -> {
-                lastCheckpoint -> {
             indexerBuilder.setLastCheckpoint(lastCheckpoint);
 
-                    logger.trace("[{}] Loaded last checkpoint [{}], looking for next checkpoint", transformId,
+            logger.trace("[{}] Loaded last checkpoint [{}], looking for next checkpoint", transformId, lastCheckpoint.getCheckpoint());
-                            lastCheckpoint.getCheckpoint());
+            transformsConfigManager
-                    transformsConfigManager.getTransformCheckpoint(transformId, lastCheckpoint.getCheckpoint() + 1,
+                .getTransformCheckpoint(transformId, lastCheckpoint.getCheckpoint() + 1, getTransformNextCheckpointListener);
-                            getTransformNextCheckpointListener);
+        }, error -> {
-                },
-                error -> {
             String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_LOAD_TRANSFORM_CHECKPOINT, transformId);
             logger.error(msg, error);
             markAsFailed(buildTask, msg);
-                }
+        });
-        );
 
         // <5> Set the previous stats (if they exist), initialize the indexer, start the task (If it is STOPPED)
         // Since we don't create the task until `_start` is called, if we see that the task state is stopped, attempt to start
         // Schedule execution regardless
-        ActionListener<Tuple<TransformStoredDoc, SeqNoPrimaryTermAndIndex>> transformStatsActionListener = ActionListener.wrap(
+        ActionListener<Tuple<TransformStoredDoc, SeqNoPrimaryTermAndIndex>> transformStatsActionListener = ActionListener
-            stateAndStatsAndSeqNoPrimaryTermAndIndex -> {
+            .wrap(stateAndStatsAndSeqNoPrimaryTermAndIndex -> {
                 TransformStoredDoc stateAndStats = stateAndStatsAndSeqNoPrimaryTermAndIndex.v1();
                 SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex = stateAndStatsAndSeqNoPrimaryTermAndIndex.v2();
                 // Since we have not set the value for this yet, it SHOULD be null
-                buildTask.updateSeqNoPrimaryTermAndIndex(null, seqNoPrimaryTermAndIndex);
                 logger.trace("[{}] initializing state and stats: [{}]", transformId, stateAndStats.toString());
                 TransformState transformState = stateAndStats.getTransformState();
-                indexerBuilder.setInitialStats(stateAndStats.getTransformStats())
+                indexerBuilder
+                    .setInitialStats(stateAndStats.getTransformStats())
                     .setInitialPosition(stateAndStats.getTransformState().getPosition())
                     .setProgress(stateAndStats.getTransformState().getProgress())
                     .setIndexerState(currentIndexerState(transformState))
+                    .setSeqNoPrimaryTermAndIndex(seqNoPrimaryTermAndIndex)
                     .setShouldStopAtCheckpoint(transformState.shouldStopAtNextCheckpoint());
-                logger.debug("[{}] Loading existing state: [{}], position [{}]",
+                logger
+                    .debug(
+                        "[{}] Loading existing state: [{}], position [{}]",
                         transformId,
                         stateAndStats.getTransformState(),
-                    stateAndStats.getTransformState().getPosition());
+                        stateAndStats.getTransformState().getPosition()
+                    );
 
                 stateHolder.set(transformState);
                 final long lastCheckpoint = stateHolder.get().getCheckpoint();
@@ -238,8 +243,7 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
                     logger.trace("[{}] Restore last checkpoint: [{}]", transformId, lastCheckpoint);
                     transformsConfigManager.getTransformCheckpoint(transformId, lastCheckpoint, getTransformLastCheckpointListener);
                 }
-            },
+            }, error -> {
-            error -> {
                 if (error instanceof ResourceNotFoundException == false) {
                     String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_LOAD_TRANSFORM_STATE, transformId);
                     logger.error(msg, error);
@@ -248,50 +252,43 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
                     logger.trace("[{}] No stats found (new transform), starting the task", transformId);
                     startTask(buildTask, indexerBuilder, null, startTaskListener);
                 }
-            }
+            });
-        );
 
         // <4> set fieldmappings for the indexer, get the previous stats (if they exist)
-        ActionListener<Map<String, String>> getFieldMappingsListener = ActionListener.wrap(
+        ActionListener<Map<String, String>> getFieldMappingsListener = ActionListener.wrap(fieldMappings -> {
-            fieldMappings -> {
             indexerBuilder.setFieldMappings(fieldMappings);
             transformsConfigManager.getTransformStoredDoc(transformId, transformStatsActionListener);
-            },
+        }, error -> {
-            error -> {
+            String msg = TransformMessages
-                String msg = TransformMessages.getMessage(TransformMessages.UNABLE_TO_GATHER_FIELD_MAPPINGS,
+                .getMessage(
-                    indexerBuilder.getTransformConfig().getDestination().getIndex());
+                    TransformMessages.UNABLE_TO_GATHER_FIELD_MAPPINGS,
+                    indexerBuilder.getTransformConfig().getDestination().getIndex()
+                );
             logger.error(msg, error);
             markAsFailed(buildTask, msg);
-            }
+        });
-        );
 
         // <3> Validate the transform, assigning it to the indexer, and get the field mappings
-        ActionListener<TransformConfig> getTransformConfigListener = ActionListener.wrap(
+        ActionListener<TransformConfig> getTransformConfigListener = ActionListener.wrap(config -> {
-            config -> {
             if (config.isValid()) {
                 indexerBuilder.setTransformConfig(config);
                 SchemaUtil.getDestinationFieldMappings(client, config.getDestination().getIndex(), getFieldMappingsListener);
             } else {
-                    markAsFailed(buildTask,
+                markAsFailed(buildTask, TransformMessages.getMessage(TransformMessages.TRANSFORM_CONFIGURATION_INVALID, transformId));
-                        TransformMessages.getMessage(TransformMessages.TRANSFORM_CONFIGURATION_INVALID, transformId));
             }
-            },
+        }, error -> {
-            error -> {
             String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_LOAD_TRANSFORM_CONFIGURATION, transformId);
             logger.error(msg, error);
             markAsFailed(buildTask, msg);
-            }
+        });
-        );
 
         // <2> Get the transform config
-        ActionListener<Void> templateCheckListener = ActionListener.wrap(
+        ActionListener<Void> templateCheckListener = ActionListener
-            aVoid -> transformsConfigManager.getTransformConfiguration(transformId, getTransformConfigListener),
+            .wrap(aVoid -> transformsConfigManager.getTransformConfiguration(transformId, getTransformConfigListener), error -> {
-            error -> {
                 String msg = "Failed to create internal index mappings";
                 logger.error(msg, error);
                 markAsFailed(buildTask, msg);
-            }
+            });
-        );
 
         // <1> Check the internal index template is installed
         TransformInternalIndex.installLatestVersionedIndexTemplateIfRequired(clusterService, client, templateCheckListener);
@@ -320,10 +317,15 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
     private void markAsFailed(TransformTask task, String reason) {
         CountDownLatch latch = new CountDownLatch(1);
 
-        task.markAsFailed(reason, new LatchedActionListener<>(ActionListener.wrap(
+        task
-            nil -> {},
+            .fail(
-            failure -> logger.error("Failed to set task [" + task.getTransformId() +"] to failed", failure)
+                reason,
-        ), latch));
+                new LatchedActionListener<>(
+                    ActionListener
+                        .wrap(nil -> {}, failure -> logger.error("Failed to set task [" + task.getTransformId() + "] to failed", failure)),
+                    latch
+                )
+            );
         try {
             latch.await(MARK_AS_FAILED_TIMEOUT_SEC, TimeUnit.SECONDS);
         } catch (InterruptedException e) {
@@ -331,10 +333,12 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
         }
     }
 
-    private void startTask(TransformTask buildTask,
+    private void startTask(
+        TransformTask buildTask,
         ClientTransformIndexerBuilder indexerBuilder,
         Long previousCheckpoint,
-                           ActionListener<StartTransformAction.Response> listener) {
+        ActionListener<StartTransformAction.Response> listener
+    ) {
         buildTask.initializeIndexer(indexerBuilder);
         // TransformTask#start will fail if the task state is FAILED
         buildTask.setNumFailureRetries(numFailureRetries).start(previousCheckpoint, listener);
@@ -345,9 +349,25 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
     }
 
     @Override
-    protected AllocatedPersistentTask createTask(long id, String type, String action, TaskId parentTaskId,
+    protected AllocatedPersistentTask createTask(
-            PersistentTasksCustomMetaData.PersistentTask<TransformTaskParams> persistentTask, Map<String, String> headers) {
+        long id,
-        return new TransformTask(id, type, action, parentTaskId, persistentTask.getParams(),
+        String type,
-            (TransformState) persistentTask.getState(), schedulerEngine, auditor, threadPool, headers);
+        String action,
+        TaskId parentTaskId,
+        PersistentTasksCustomMetaData.PersistentTask<TransformTaskParams> persistentTask,
+        Map<String, String> headers
+    ) {
+        return new TransformTask(
+            id,
+            type,
+            action,
+            parentTaskId,
+            persistentTask.getParams(),
+            (TransformState) persistentTask.getState(),
+            schedulerEngine,
+            auditor,
+            threadPool,
+            headers
+        );
     }
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformProgressGatherer.java

@@ -26,53 +26,63 @@ import java.util.function.Function;
  */
 public final class TransformProgressGatherer {
 
+    private Client client;
+
+    TransformProgressGatherer(Client client) {
+        this.client = client;
+    }
+
     /**
      * This gathers the total docs given the config and search
      *
-     * @param client ES Client to make queries
      * @param filterQuery The adapted filter that can optionally take into account checkpoint information
      * @param config The transform config containing headers, source, pivot, etc. information
      * @param progressListener The listener to notify when progress object has been created
      */
-    public static void getInitialProgress(Client client,
+    public void getInitialProgress(QueryBuilder filterQuery, TransformConfig config, ActionListener<TransformProgress> progressListener) {
-                                          QueryBuilder filterQuery,
-                                          TransformConfig config,
-                                          ActionListener<TransformProgress> progressListener) {
         SearchRequest request = getSearchRequest(config, filterQuery);
 
-        ActionListener<SearchResponse> searchResponseActionListener = ActionListener.wrap(
+        ActionListener<SearchResponse> searchResponseActionListener = ActionListener
+            .wrap(
                 searchResponse -> progressListener.onResponse(searchResponseToTransformProgressFunction().apply(searchResponse)),
                 progressListener::onFailure
             );
-        ClientHelper.executeWithHeadersAsync(config.getHeaders(),
+        ClientHelper
+            .executeWithHeadersAsync(
+                config.getHeaders(),
                 ClientHelper.TRANSFORM_ORIGIN,
                 client,
                 SearchAction.INSTANCE,
                 request,
-            searchResponseActionListener);
+                searchResponseActionListener
+            );
     }
 
     public static SearchRequest getSearchRequest(TransformConfig config, QueryBuilder filteredQuery) {
         SearchRequest request = new SearchRequest(config.getSource().getIndex());
         request.allowPartialSearchResults(false);
         BoolQueryBuilder existsClauses = QueryBuilders.boolQuery();
-        config.getPivotConfig()
+        config
+            .getPivotConfig()
             .getGroupConfig()
             .getGroups()
             .values()
             // TODO change once we allow missing_buckets
             .forEach(src -> existsClauses.must(QueryBuilders.existsQuery(src.getField())));
 
-        request.source(new SearchSourceBuilder()
+        request
+            .source(
+                new SearchSourceBuilder()
                     .size(0)
                     .trackTotalHits(true)
-            .query(QueryBuilders.boolQuery()
+                    .query(QueryBuilders.boolQuery().filter(filteredQuery).filter(existsClauses))
-                .filter(filteredQuery)
+            );
-                .filter(existsClauses)));
         return request;
     }
 
     public static Function<SearchResponse, TransformProgress> searchResponseToTransformProgressFunction() {
-        return searchResponse -> new TransformProgress(searchResponse.getHits().getTotalHits().value, 0L, 0L);
+        return searchResponse -> searchResponse != null
+            ? new TransformProgress(searchResponse.getHits().getTotalHits().value, 0L, 0L)
+            : null;
     }
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java

@@ -13,8 +13,6 @@ import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
@@ -27,40 +25,26 @@ import org.elasticsearch.xpack.core.scheduler.SchedulerEngine.Event;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
+import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo;
 import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
 import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
-import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
-import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo;
 import org.elasticsearch.xpack.core.transform.transforms.TransformState;
+import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
 import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
 import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
-import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex;
 
 import java.util.Arrays;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.xpack.core.transform.TransformMessages.CANNOT_START_FAILED_TRANSFORM;
 import static org.elasticsearch.xpack.core.transform.TransformMessages.CANNOT_STOP_FAILED_TRANSFORM;
 
-
+public class TransformTask extends AllocatedPersistentTask implements SchedulerEngine.Listener, TransformContext.Listener {
-public class TransformTask extends AllocatedPersistentTask implements SchedulerEngine.Listener {
 
     // Default interval the scheduler sends an event if the config does not specify a frequency
     private static final long SCHEDULER_NEXT_MILLISECONDS = 60000;
     private static final Logger logger = LogManager.getLogger(TransformTask.class);
-    private static final int DEFAULT_FAILURE_RETRIES = 10;
-    private volatile int numFailureRetries = DEFAULT_FAILURE_RETRIES;
-    // How many times the transform task can retry on an non-critical failure
-    public static final Setting<Integer> NUM_FAILURE_RETRIES_SETTING = Setting.intSetting(
-        "xpack.transform.num_transform_failure_retries",
-        DEFAULT_FAILURE_RETRIES,
-        0,
-        100,
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic);
     private static final IndexerState[] RUNNING_STATES = new IndexerState[] { IndexerState.STARTED, IndexerState.INDEXING };
     public static final String SCHEDULE_NAME = TransformField.TASK_NAME + "/schedule";
 
@@ -70,19 +54,21 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
     private final TransformAuditor auditor;
     private final TransformIndexerPosition initialPosition;
     private final IndexerState initialIndexerState;
-
+    private final TransformContext context;
     private final SetOnce<ClientTransformIndexer> indexer = new SetOnce<>();
 
-    private final AtomicReference<TransformTaskState> taskState;
+    public TransformTask(
-    private final AtomicReference<String> stateReason;
+        long id,
-    private final AtomicReference<SeqNoPrimaryTermAndIndex> seqNoPrimaryTermAndIndex = new AtomicReference<>(null);
+        String type,
-    // the checkpoint of this transform, storing the checkpoint until data indexing from source to dest is _complete_
+        String action,
-    // Note: Each indexer run creates a new future checkpoint which becomes the current checkpoint only after the indexer run finished
+        TaskId parentTask,
-    private final AtomicLong currentCheckpoint;
+        TransformTaskParams transform,
-
+        TransformState state,
-    public TransformTask(long id, String type, String action, TaskId parentTask, TransformTaskParams transform,
+        SchedulerEngine schedulerEngine,
-                                  TransformState state, SchedulerEngine schedulerEngine, TransformAuditor auditor,
+        TransformAuditor auditor,
-                                  ThreadPool threadPool, Map<String, String> headers) {
+        ThreadPool threadPool,
+        Map<String, String> headers
+    ) {
         super(id, type, action, TransformField.PERSISTENT_TASK_DESCRIPTION_PREFIX + transform.getId(), parentTask, headers);
         this.transform = transform;
         this.schedulerEngine = schedulerEngine;
@@ -91,8 +77,9 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
         IndexerState initialState = IndexerState.STOPPED;
         TransformTaskState initialTaskState = TransformTaskState.STOPPED;
         String initialReason = null;
-        long initialGeneration = 0;
+        long initialCheckpoint = 0;
         TransformIndexerPosition initialPosition = null;
+
         if (state != null) {
             initialTaskState = state.getTaskState();
             initialReason = state.getReason();
@@ -107,14 +94,13 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
                 initialState = existingState;
             }
             initialPosition = state.getPosition();
-            initialGeneration = state.getCheckpoint();
+            initialCheckpoint = state.getCheckpoint();
         }
 
         this.initialIndexerState = initialState;
         this.initialPosition = initialPosition;
-        this.currentCheckpoint = new AtomicLong(initialGeneration);
+
-        this.taskState = new AtomicReference<>(initialTaskState);
+        this.context = new TransformContext(initialTaskState, initialReason, initialCheckpoint, this);
-        this.stateReason = new AtomicReference<>(initialReason);
     }
 
     public String getTransformId() {
@@ -136,24 +122,26 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
     public TransformState getState() {
         if (getIndexer() == null) {
             return new TransformState(
-                taskState.get(),
+                context.getTaskState(),
                 initialIndexerState,
                 initialPosition,
-                currentCheckpoint.get(),
+                context.getCheckpoint(),
-                stateReason.get(),
+                context.getStateReason(),
                 null,
                 null,
-            false);
+                false
+            );
         } else {
             return new TransformState(
-               taskState.get(),
+                context.getTaskState(),
                 indexer.get().getState(),
                 indexer.get().getPosition(),
-               currentCheckpoint.get(),
+                context.getCheckpoint(),
-               stateReason.get(),
+                context.getStateReason(),
                 getIndexer().getProgress(),
                 null,
-               getIndexer().shouldStopAtCheckpoint());
+                context.shouldStopAtCheckpoint()
+            );
         }
     }
 
@@ -165,138 +153,149 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
         }
     }
 
-    public long getCheckpoint() {
+    public void getCheckpointingInfo(
-        return currentCheckpoint.get();
+        TransformCheckpointService transformsCheckpointService,
-    }
+        ActionListener<TransformCheckpointingInfo> listener
-
+    ) {
-    long incrementCheckpoint() {
-        return currentCheckpoint.getAndIncrement();
-    }
-
-    public void getCheckpointingInfo(TransformCheckpointService transformsCheckpointService,
-            ActionListener<TransformCheckpointingInfo> listener) {
         ClientTransformIndexer indexer = getIndexer();
         if (indexer == null) {
-            transformsCheckpointService.getCheckpointingInfo(
+            transformsCheckpointService.getCheckpointingInfo(transform.getId(), context.getCheckpoint(), initialPosition, null, listener);
-                    transform.getId(),
-                    currentCheckpoint.get(),
-                    initialPosition,
-                    null,
-                    listener);
             return;
         }
-        indexer.getCheckpointProvider().getCheckpointingInfo(
+        indexer
+            .getCheckpointProvider()
+            .getCheckpointingInfo(
                 indexer.getLastCheckpoint(),
                 indexer.getNextCheckpoint(),
                 indexer.getPosition(),
                 indexer.getProgress(),
-                ActionListener.wrap(
+                ActionListener.wrap(info -> {
-                    info -> {
+                    if (context.getChangesLastDetectedAt() == null) {
-                        if (indexer.getChangesLastDetectedAt() == null) {
                         listener.onResponse(info);
                     } else {
-                            listener.onResponse(info.setChangesLastDetectedAt(indexer.getChangesLastDetectedAt()));
+                        listener.onResponse(info.setChangesLastDetectedAt(context.getChangesLastDetectedAt()));
                     }
-                    },
+                }, listener::onFailure)
-                    listener::onFailure
+            );
-                ));
     }
 
     /**
      * Starts the transform and schedules it to be triggered in the future.
      *
-     * NOTE: This should ONLY be called via {@link TransformPersistentTasksExecutor}
      *
      * @param startingCheckpoint The starting checkpoint, could null. Null indicates that there is no starting checkpoint
      * @param listener The listener to alert once started
      */
     synchronized void start(Long startingCheckpoint, ActionListener<StartTransformAction.Response> listener) {
         logger.debug("[{}] start called with state [{}].", getTransformId(), getState());
-        if (taskState.get() == TransformTaskState.FAILED) {
+        if (context.getTaskState() == TransformTaskState.FAILED) {
-            listener.onFailure(new ElasticsearchStatusException(
+            listener
-                TransformMessages.getMessage(CANNOT_START_FAILED_TRANSFORM,
+                .onFailure(
-                    getTransformId(),
+                    new ElasticsearchStatusException(
-                    stateReason.get()),
+                        TransformMessages.getMessage(CANNOT_START_FAILED_TRANSFORM, getTransformId(), context.getStateReason()),
-                RestStatus.CONFLICT));
+                        RestStatus.CONFLICT
+                    )
+                );
             return;
         }
         if (getIndexer() == null) {
             // If our state is failed AND the indexer is null, the user needs to _stop?force=true so that the indexer gets
             // fully initialized.
             // If we are NOT failed, then we can assume that `start` was just called early in the process.
-            String msg = taskState.get() == TransformTaskState.FAILED ?
+            String msg = context.getTaskState() == TransformTaskState.FAILED
-                "It failed during the initialization process; force stop to allow reinitialization." :
+                ? "It failed during the initialization process; force stop to allow reinitialization."
-                "Try again later.";
+                : "Try again later.";
-            listener.onFailure(new ElasticsearchStatusException("Task for transform [{}] not fully initialized. {}",
+            listener
+                .onFailure(
+                    new ElasticsearchStatusException(
+                        "Task for transform [{}] not fully initialized. {}",
                         RestStatus.CONFLICT,
                         getTransformId(),
-                msg));
+                        msg
+                    )
+                );
             return;
         }
         final IndexerState newState = getIndexer().start();
         if (Arrays.stream(RUNNING_STATES).noneMatch(newState::equals)) {
-            listener.onFailure(new ElasticsearchException("Cannot start task for transform [{}], because state was [{}]",
+            listener
-                transform.getId(), newState));
+                .onFailure(
+                    new ElasticsearchException("Cannot start task for transform [{}], because state was [{}]", transform.getId(), newState)
+                );
             return;
         }
-        stateReason.set(null);
+        context.resetTaskState();
-        taskState.set(TransformTaskState.STARTED);
+
         if (startingCheckpoint != null) {
-            currentCheckpoint.set(startingCheckpoint);
+            context.setCheckpoint(startingCheckpoint);
         }
 
         final TransformState state = new TransformState(
             TransformTaskState.STARTED,
             IndexerState.STOPPED,
             getIndexer().getPosition(),
-            currentCheckpoint.get(),
+            context.getCheckpoint(),
             null,
             getIndexer().getProgress(),
             null,
-            getIndexer().shouldStopAtCheckpoint());
+            context.shouldStopAtCheckpoint()
+        );
 
         logger.info("[{}] updating state for transform to [{}].", transform.getId(), state.toString());
         // Even though the indexer information is persisted to an index, we still need TransformTaskState in the clusterstate
         // This keeps track of STARTED, FAILED, STOPPED
         // This is because a FAILED state can occur because we cannot read the config from the internal index, which would imply that
         // we could not read the previous state information from said index.
-        persistStateToClusterState(state, ActionListener.wrap(
+        persistStateToClusterState(state, ActionListener.wrap(task -> {
-            task -> {
+            auditor.info(transform.getId(), "Updated transform state to [" + state.getTaskState() + "].");
-                auditor.info(transform.getId(),
-                    "Updated transform state to [" + state.getTaskState() + "].");
             long now = System.currentTimeMillis();
             // kick off the indexer
             triggered(new Event(schedulerJobName(), now, now));
             registerWithSchedulerJob();
             listener.onResponse(new StartTransformAction.Response(true));
-            },
+        }, exc -> {
-            exc -> {
+            auditor
-                auditor.warning(transform.getId(),
+                .warning(
-                    "Failed to persist to cluster state while marking task as started. Failure: " + exc.getMessage());
+                    transform.getId(),
+                    "Failed to persist to cluster state while marking task as started. Failure: " + exc.getMessage()
+                );
             logger.error(new ParameterizedMessage("[{}] failed updating state to [{}].", getTransformId(), state), exc);
             getIndexer().stop();
-                listener.onFailure(new ElasticsearchException("Error while updating state for transform ["
+            listener
-                    + transform.getId() + "] to [" + state.getIndexerState() + "].", exc));
+                .onFailure(
+                    new ElasticsearchException(
+                        "Error while updating state for transform [" + transform.getId() + "] to [" + state.getIndexerState() + "].",
+                        exc
+                    )
+                );
+        }));
     }
-        ));
+
+    void setShouldStopAtCheckpoint(boolean shouldStopAtCheckpoint) {
+        this.context.setShouldStopAtCheckpoint(shouldStopAtCheckpoint);
     }
 
     /**
      * This sets the flag for the task to stop at the next checkpoint.
      *
-     * If first persists the flag to cluster state, and then mutates the local variable.
+     * If first persists the flag and then mutates the local variable.
      *
-     * It only persists to cluster state if the value is different than what is currently held in memory.
+     * It only persists if the value is different than what is currently held in memory.
      * @param shouldStopAtCheckpoint whether or not we should stop at the next checkpoint or not
      * @param shouldStopAtCheckpointListener the listener to return to when we have persisted the updated value to the state index.
      */
-    public synchronized void setShouldStopAtCheckpoint(boolean shouldStopAtCheckpoint,
+    public synchronized void setShouldStopAtCheckpoint(
-                                                       ActionListener<Void> shouldStopAtCheckpointListener) {
+        boolean shouldStopAtCheckpoint,
-        logger.debug("[{}] attempted to set task to stop at checkpoint [{}] with state [{}]",
+        ActionListener<Void> shouldStopAtCheckpointListener
+    ) {
+        logger
+            .debug(
+                "[{}] attempted to set task to stop at checkpoint [{}] with state [{}]",
                 getTransformId(),
                 shouldStopAtCheckpoint,
-            getState());
+                getState()
-        if (taskState.get() != TransformTaskState.STARTED || getIndexer() == null) {
+            );
+        if (context.getTaskState() != TransformTaskState.STARTED || getIndexer() == null) {
             shouldStopAtCheckpointListener.onResponse(null);
             return;
         }
@@ -304,11 +303,14 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
     }
 
     public synchronized void stop(boolean force, boolean shouldStopAtCheckpoint) {
-        logger.debug("[{}] stop called with force [{}], shouldStopAtCheckpoint [{}], state [{}]",
+        logger
+            .debug(
+                "[{}] stop called with force [{}], shouldStopAtCheckpoint [{}], state [{}]",
                 getTransformId(),
                 force,
                 shouldStopAtCheckpoint,
-            getState());
+                getState()
+            );
         if (getIndexer() == null) {
             // If there is no indexer the task has not been triggered
             // but it still needs to be stopped and removed
@@ -320,17 +322,17 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
             return;
         }
 
-        if (taskState.get() == TransformTaskState.FAILED && force == false) {
+        if (context.getTaskState() == TransformTaskState.FAILED && force == false) {
             throw new ElasticsearchStatusException(
-                TransformMessages.getMessage(CANNOT_STOP_FAILED_TRANSFORM,
+                TransformMessages.getMessage(CANNOT_STOP_FAILED_TRANSFORM, getTransformId(), context.getStateReason()),
-                    getTransformId(),
+                RestStatus.CONFLICT
-                    stateReason.get()),
+            );
-                RestStatus.CONFLICT);
         }
 
-        stateReason.set(null);
+        context.resetReasonAndFailureCounter();
+
         // No reason to keep it in the potentially failed state.
-        boolean wasFailed = taskState.compareAndSet(TransformTaskState.FAILED, TransformTaskState.STARTED);
+        boolean wasFailed = context.setTaskState(TransformTaskState.FAILED, TransformTaskState.STARTED);
         // shouldStopAtCheckpoint only comes into play when onFinish is called (or doSaveState right after).
         // if it is false, stop immediately
         if (shouldStopAtCheckpoint == false ||
@@ -360,18 +362,21 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
             return;
         }
 
-        if (taskState.get() == TransformTaskState.FAILED || taskState.get() == TransformTaskState.STOPPED) {
+        if (context.getTaskState() == TransformTaskState.FAILED || context.getTaskState() == TransformTaskState.STOPPED) {
-            logger.debug("[{}] schedule was triggered for transform but task is [{}]. Ignoring trigger.",
+            logger
+                .debug(
+                    "[{}] schedule was triggered for transform but task is [{}]. Ignoring trigger.",
                     getTransformId(),
-                taskState.get());
+                    context.getTaskState()
+                );
             return;
         }
 
         // ignore trigger if indexer is running or completely stopped
         IndexerState indexerState = getIndexer().getState();
-        if (IndexerState.INDEXING.equals(indexerState) ||
+        if (IndexerState.INDEXING.equals(indexerState)
-            IndexerState.STOPPING.equals(indexerState) ||
+            || IndexerState.STOPPING.equals(indexerState)
-            IndexerState.STOPPED.equals(indexerState)) {
+            || IndexerState.STOPPED.equals(indexerState)) {
             logger.debug("[{}] indexer for transform has state [{}]. Ignoring trigger.", getTransformId(), indexerState);
             return;
         }
@@ -379,7 +384,7 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
         logger.debug("[{}] transform indexer schedule has triggered, state: [{}].", event.getJobName(), indexerState);
 
         // if it runs for the 1st time we just do it, if not we check for changes
-        if (currentCheckpoint.get() == 0) {
+        if (context.getCheckpoint() == 0) {
             logger.debug("[{}] trigger initial run.", getTransformId());
             getIndexer().maybeTriggerAsyncJob(System.currentTimeMillis());
         } else if (getIndexer().isContinuous()) {
@@ -391,31 +396,27 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
      * Attempt to gracefully cleanup the transform so it can be terminated.
      * This tries to remove the job from the scheduler and completes the persistent task
      */
-    synchronized void shutdown() {
+    @Override
+    public synchronized void shutdown() {
         deregisterSchedulerJob();
         markAsCompleted();
     }
 
-    void persistStateToClusterState(TransformState state,
+    void persistStateToClusterState(TransformState state, ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
-                                    ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> listener) {
+        updatePersistentTaskState(state, ActionListener.wrap(success -> {
-        updatePersistentTaskState(state, ActionListener.wrap(
-            success -> {
             logger.debug("[{}] successfully updated state for transform to [{}].", transform.getId(), state.toString());
             listener.onResponse(success);
-            },
+        }, failure -> {
-            failure -> {
+            logger.error(new ParameterizedMessage("[{}] failed to update cluster state for transform.", transform.getId()), failure);
-                logger.error(new ParameterizedMessage("[{}] failed to update cluster state for transform.",
-                    transform.getId()),
-                    failure);
             listener.onFailure(failure);
-            }
+        }));
-        ));
     }
 
-    synchronized void markAsFailed(String reason, ActionListener<Void> listener) {
+    @Override
+    public synchronized void fail(String reason, ActionListener<Void> listener) {
         // If we are already flagged as failed, this probably means that a second trigger started firing while we were attempting to
         // flag the previously triggered indexer as failed. Exit early as we are already flagged as failed.
-        if (taskState.get() == TransformTaskState.FAILED) {
+        if (context.getTaskState() == TransformTaskState.FAILED) {
             logger.warn("[{}] is already failed but encountered new failure; reason [{}].", getTransformId(), reason);
             listener.onResponse(null);
             return;
@@ -441,28 +442,21 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
         deregisterSchedulerJob();
         // The idea of stopping at the next checkpoint is no longer valid. Since a failed task could potentially START again,
         // we should set this flag to false.
-        if (getIndexer() != null) {
+        context.setShouldStopAtCheckpoint(false);
-            getIndexer().setShouldStopAtCheckpoint(false);
+
-        }
         // The end user should see that the task is in a failed state, and attempt to stop it again but with force=true
-        taskState.set(TransformTaskState.FAILED);
+        context.setTaskStateToFailed(reason);
-        stateReason.set(reason);
         TransformState newState = getState();
         // Even though the indexer information is persisted to an index, we still need TransformTaskState in the clusterstate
         // This keeps track of STARTED, FAILED, STOPPED
         // This is because a FAILED state could occur because we failed to read the config from the internal index, which would imply that
         // we could not read the previous state information from said index.
-        persistStateToClusterState(newState, ActionListener.wrap(
+        persistStateToClusterState(newState, ActionListener.wrap(r -> listener.onResponse(null), e -> {
-            r -> listener.onResponse(null),
-            e -> {
             String msg = "Failed to persist to cluster state while marking task as failed with reason [" + reason + "].";
-                auditor.warning(transform.getId(),
+            auditor.warning(transform.getId(), msg + " Failure: " + e.getMessage());
-                    msg + " Failure: " + e.getMessage());
+            logger.error(new ParameterizedMessage("[{}] {}", getTransformId(), msg), e);
-                logger.error(new ParameterizedMessage("[{}] {}", getTransformId(), msg),
-                    e);
             listener.onFailure(e);
-            }
+        }));
-        ));
     }
 
     /**
@@ -472,9 +466,7 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
      */
     @Override
     public synchronized void onCancelled() {
-        logger.info("[{}] received cancellation request for transform, state: [{}].",
+        logger.info("[{}] received cancellation request for transform, state: [{}].", getTransformId(), context.getTaskState());
-            getTransformId(),
-            taskState.get());
         if (getIndexer() != null && getIndexer().abort()) {
             // there is no background transform running, we can shutdown safely
             shutdown();
@@ -482,14 +474,10 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
     }
 
     TransformTask setNumFailureRetries(int numFailureRetries) {
-        this.numFailureRetries = numFailureRetries;
+        context.setNumFailureRetries(numFailureRetries);
         return this;
     }
 
-    int getNumFailureRetries() {
-        return numFailureRetries;
-    }
-
     private void registerWithSchedulerJob() {
         schedulerEngine.register(this);
         final SchedulerEngine.Job schedulerJob = new SchedulerEngine.Job(schedulerJobName(), next());
@@ -513,20 +501,7 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
     }
 
     synchronized void initializeIndexer(ClientTransformIndexerBuilder indexerBuilder) {
-        indexer.set(indexerBuilder.build(this));
+        indexer.set(indexerBuilder.build(getThreadPool().executor(ThreadPool.Names.GENERIC), context));
-    }
-
-    void updateSeqNoPrimaryTermAndIndex(SeqNoPrimaryTermAndIndex expectedValue, SeqNoPrimaryTermAndIndex newValue) {
-        boolean updated = seqNoPrimaryTermAndIndex.compareAndSet(expectedValue, newValue);
-        // This should never happen. We ONLY ever update this value if at initialization or we just finished updating the document
-        // famous last words...
-        assert updated :
-            "[" + getTransformId() + "] unexpected change to seqNoPrimaryTermAndIndex.";
-    }
-
-    @Nullable
-    SeqNoPrimaryTermAndIndex getSeqNoPrimaryTermAndIndex() {
-        return seqNoPrimaryTermAndIndex.get();
     }
 
     ThreadPool getThreadPool() {
@@ -534,14 +509,7 @@ public class TransformTask extends AllocatedPersistentTask implements SchedulerE
     }
 
     TransformTaskState getTaskState() {
-        return taskState.get();
+        return context.getTaskState();
     }
 
-    void setStateReason(String reason) {
-        stateReason.set(reason);
-    }
-
-    String getStateReason() {
-        return stateReason.get();
-    }
 }

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProviderTests.java

@@ -50,33 +50,40 @@ public class DefaultCheckpointProviderTests extends ESTestCase {
             client,
             transformConfigManager,
             transformAuditor,
-            transformConfig);
+            transformConfig
+        );
 
         assertExpectation(
-            new MockLogAppender.SeenEventExpectation("warn when source is empty",
+            new MockLogAppender.SeenEventExpectation(
+                "warn when source is empty",
                 checkpointProviderlogger.getName(),
                 Level.WARN,
-                "Source did not resolve to any open indexes for transform [" + transformId + "]"),
+                "[" + transformId + "] Source did not resolve to any open indexes"
-            new MockTransformAuditor.SeenAuditExpectation("warn when source is empty",
+            ),
+            new MockTransformAuditor.SeenAuditExpectation(
+                "warn when source is empty",
                 org.elasticsearch.xpack.core.common.notifications.Level.WARNING,
                 transformId,
-                "Source did not resolve to any open indexes"),
+                "Source did not resolve to any open indexes"
-            () -> {
+            ),
-                    provider.reportSourceIndexChanges(Collections.singleton("index"), Collections.emptySet());
+            () -> { provider.reportSourceIndexChanges(Collections.singleton("index"), Collections.emptySet()); }
-                });
+        );
 
         assertExpectation(
-            new MockLogAppender.UnseenEventExpectation("do not warn if empty again",
+            new MockLogAppender.UnseenEventExpectation(
+                "do not warn if empty again",
                 checkpointProviderlogger.getName(),
                 Level.WARN,
-                "Source did not resolve to any concrete indexes"),
+                "Source did not resolve to any concrete indexes"
-            new MockTransformAuditor.UnseenAuditExpectation("do not warn if empty again",
+            ),
+            new MockTransformAuditor.UnseenAuditExpectation(
+                "do not warn if empty again",
                 org.elasticsearch.xpack.core.common.notifications.Level.WARNING,
                 transformId,
-                "Source did not resolve to any concrete indexes"),
+                "Source did not resolve to any concrete indexes"
-            () -> {
+            ),
-                    provider.reportSourceIndexChanges(Collections.emptySet(), Collections.emptySet());
+            () -> { provider.reportSourceIndexChanges(Collections.emptySet(), Collections.emptySet()); }
-                });
+        );
     }
 
     public void testReportSourceIndexChangesAddDelete() throws Exception {
@@ -87,48 +94,55 @@ public class DefaultCheckpointProviderTests extends ESTestCase {
             client,
             transformConfigManager,
             transformAuditor,
-            transformConfig);
+            transformConfig
+        );
 
         assertExpectation(
-            new MockLogAppender.SeenEventExpectation("info about adds/removal",
+            new MockLogAppender.SeenEventExpectation(
+                "info about adds/removal",
                 checkpointProviderlogger.getName(),
                 Level.DEBUG,
-                "Source index resolve found changes, removedIndexes: [index], new indexes: [other_index] for transform [" +
+                "[" + transformId + "] Source index resolve found changes, removedIndexes: [index], new indexes: [other_index]"
-                    transformId + "]"),
+            ),
-            new MockTransformAuditor.SeenAuditExpectation("info about adds/removal",
+            new MockTransformAuditor.SeenAuditExpectation(
+                "info about adds/removal",
                 org.elasticsearch.xpack.core.common.notifications.Level.INFO,
                 transformId,
-                "Source index resolve found changes, removedIndexes: [index], new indexes: [other_index]"),
+                "Source index resolve found changes, removedIndexes: [index], new indexes: [other_index]"
-            () -> {
+            ),
-                    provider.reportSourceIndexChanges(Collections.singleton("index"), Collections.singleton("other_index"));
+            () -> { provider.reportSourceIndexChanges(Collections.singleton("index"), Collections.singleton("other_index")); }
-                });
+        );
 
         assertExpectation(
-            new MockLogAppender.SeenEventExpectation("info about adds/removal",
+            new MockLogAppender.SeenEventExpectation(
+                "info about adds/removal",
                 checkpointProviderlogger.getName(),
                 Level.DEBUG,
-                "Source index resolve found changes, removedIndexes: [index], new indexes: [] for transform [" +
+                "[" + transformId + "] Source index resolve found changes, removedIndexes: [index], new indexes: []"
-                    transformId + "]"),
+            ),
-            new MockTransformAuditor.SeenAuditExpectation("info about adds/removal",
+            new MockTransformAuditor.SeenAuditExpectation(
+                "info about adds/removal",
                 org.elasticsearch.xpack.core.common.notifications.Level.INFO,
                 transformId,
-                "Source index resolve found changes, removedIndexes: [index], new indexes: []"),
+                "Source index resolve found changes, removedIndexes: [index], new indexes: []"
-            () -> {
+            ),
-                    provider.reportSourceIndexChanges(Sets.newHashSet("index", "other_index"), Collections.singleton("other_index"));
+            () -> { provider.reportSourceIndexChanges(Sets.newHashSet("index", "other_index"), Collections.singleton("other_index")); }
-                });
+        );
         assertExpectation(
-            new MockLogAppender.SeenEventExpectation("info about adds/removal",
+            new MockLogAppender.SeenEventExpectation(
+                "info about adds/removal",
                 checkpointProviderlogger.getName(),
                 Level.DEBUG,
-                "Source index resolve found changes, removedIndexes: [], new indexes: [other_index] for transform [" +
+                "[" + transformId + "] Source index resolve found changes, removedIndexes: [], new indexes: [other_index]"
-                    transformId + "]"),
+            ),
-            new MockTransformAuditor.SeenAuditExpectation("info about adds/removal",
+            new MockTransformAuditor.SeenAuditExpectation(
+                "info about adds/removal",
                 org.elasticsearch.xpack.core.common.notifications.Level.INFO,
                 transformId,
-                "Source index resolve found changes, removedIndexes: [], new indexes: [other_index]"),
+                "Source index resolve found changes, removedIndexes: [], new indexes: [other_index]"
-            () -> {
+            ),
-                    provider.reportSourceIndexChanges(Collections.singleton("index"), Sets.newHashSet("index", "other_index"));
+            () -> { provider.reportSourceIndexChanges(Collections.singleton("index"), Sets.newHashSet("index", "other_index")); }
-                });
+        );
     }
 
     public void testReportSourceIndexChangesAddDeleteMany() throws Exception {
@@ -139,7 +153,8 @@ public class DefaultCheckpointProviderTests extends ESTestCase {
             client,
             transformConfigManager,
             transformAuditor,
-            transformConfig);
+            transformConfig
+        );
 
         HashSet<String> oldSet = new HashSet<>();
         for (int i = 0; i < 100; ++i) {
@@ -151,23 +166,24 @@ public class DefaultCheckpointProviderTests extends ESTestCase {
         }
 
         assertExpectation(
-            new MockLogAppender.SeenEventExpectation("info about adds/removal",
+            new MockLogAppender.SeenEventExpectation(
+                "info about adds/removal",
                 checkpointProviderlogger.getName(),
                 Level.DEBUG,
-                "Source index resolve found more than 10 changes, [50] removed indexes, [50] new indexes for transform [" +
+                "[" + transformId + "] Source index resolve found more than 10 changes, [50] removed indexes, [50] new indexes"
-                    transformId + "]"),
+            ),
-            new MockTransformAuditor.SeenAuditExpectation("info about adds/removal",
+            new MockTransformAuditor.SeenAuditExpectation(
+                "info about adds/removal",
                 org.elasticsearch.xpack.core.common.notifications.Level.INFO,
                 transformId,
-                "Source index resolve found more than 10 changes, [50] removed indexes, [50] new indexes"),
+                "Source index resolve found more than 10 changes, [50] removed indexes, [50] new indexes"
-            () -> {
+            ),
-                    provider.reportSourceIndexChanges(oldSet, newSet);
+            () -> { provider.reportSourceIndexChanges(oldSet, newSet); }
-                });
+        );
     }
 
-    private void assertExpectation(LoggingExpectation loggingExpectation,
+    private void assertExpectation(LoggingExpectation loggingExpectation, AuditExpectation auditExpectation, Runnable codeBlock)
-                                   AuditExpectation auditExpectation,
+        throws IllegalAccessException {
-                                   Runnable codeBlock) throws IllegalAccessException {
         MockLogAppender mockLogAppender = new MockLogAppender();
         mockLogAppender.start();
 

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java

@@ -7,22 +7,22 @@
 package org.elasticsearch.xpack.transform.transforms;
 
 import org.elasticsearch.client.Client;
-import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
-import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
-import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
+import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
+import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex;
 import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
 
 import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -36,19 +36,12 @@ public class ClientTransformIndexerTests extends ESTestCase {
     public void testAudiOnFinishFrequency() {
         ThreadPool threadPool = mock(ThreadPool.class);
         when(threadPool.executor("generic")).thenReturn(mock(ExecutorService.class));
-        TransformTask parentTask = new TransformTask(1,
+
-            "transform",
-            "ptask",
-            new TaskId("transform:1"),
-            mock(TransformTaskParams.class),
-            null,
-            mock(SchedulerEngine.class),
-            mock(TransformAuditor.class),
-            threadPool,
-            Collections.emptyMap());
         ClientTransformIndexer indexer = new ClientTransformIndexer(
+            mock(Executor.class),
             mock(TransformConfigManager.class),
             mock(CheckpointProvider.class),
+            new TransformProgressGatherer(mock(Client.class)),
             new AtomicReference<>(IndexerState.STOPPED),
             null,
             mock(Client.class),
@@ -57,18 +50,12 @@ public class ClientTransformIndexerTests extends ESTestCase {
             mock(TransformConfig.class),
             Collections.emptyMap(),
             null,
-            new TransformCheckpoint("transform",
+            new TransformCheckpoint("transform", Instant.now().toEpochMilli(), 0L, Collections.emptyMap(), Instant.now().toEpochMilli()),
-                Instant.now().toEpochMilli(),
+            new TransformCheckpoint("transform", Instant.now().toEpochMilli(), 2L, Collections.emptyMap(), Instant.now().toEpochMilli()),
-                0L,
+            new SeqNoPrimaryTermAndIndex(1, 1, TransformInternalIndexConstants.LATEST_INDEX_NAME),
-                Collections.emptyMap(),
+            mock(TransformContext.class),
-                Instant.now().toEpochMilli()),
+            false
-            new TransformCheckpoint("transform",
+        );
-                Instant.now().toEpochMilli(),
-                2L,
-                Collections.emptyMap(),
-                Instant.now().toEpochMilli()),
-            parentTask,
-            false);
 
         List<Boolean> shouldAudit = IntStream.range(0, 100_000).boxed().map(indexer::shouldAuditOnFinish).collect(Collectors.toList());
 

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java

@@ -18,26 +18,28 @@ import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.breaker.CircuitBreaker.Durability;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.test.client.NoOpClient;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
 import org.elasticsearch.xpack.core.indexing.IterationResult;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
-import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
+import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
+import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.AggregationConfigTests;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfigTests;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig;
+import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
+import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
 import org.elasticsearch.xpack.transform.transforms.pivot.Pivot;
+import org.junit.After;
 import org.junit.Before;
 
 import java.io.PrintWriter;
@@ -48,6 +50,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -63,7 +66,6 @@ import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
 public class TransformIndexerTests extends ESTestCase {
 
@@ -80,17 +82,36 @@ public class TransformIndexerTests extends ESTestCase {
 
         MockedTransformIndexer(
             Executor executor,
+            TransformConfigManager transformsConfigManager,
+            CheckpointProvider checkpointProvider,
+            TransformProgressGatherer progressGatherer,
             TransformConfig transformConfig,
             Map<String, String> fieldMappings,
             TransformAuditor auditor,
             AtomicReference<IndexerState> initialState,
             TransformIndexerPosition initialPosition,
             TransformIndexerStats jobStats,
+            TransformContext context,
             Function<SearchRequest, SearchResponse> searchFunction,
             Function<BulkRequest, BulkResponse> bulkFunction,
-                Consumer<Exception> failureConsumer) {
+            Consumer<Exception> failureConsumer
-            super(executor, auditor, transformConfig, fieldMappings, initialState, initialPosition, jobStats,
+        ) {
-                    /* TransformProgress */ null, TransformCheckpoint.EMPTY, TransformCheckpoint.EMPTY);
+            super(
+                executor,
+                transformsConfigManager,
+                checkpointProvider,
+                progressGatherer,
+                auditor,
+                transformConfig,
+                fieldMappings,
+                initialState,
+                initialPosition,
+                jobStats,
+                /* TransformProgress */ null,
+                TransformCheckpoint.EMPTY,
+                TransformCheckpoint.EMPTY,
+                context
+            );
             this.searchFunction = searchFunction;
             this.bulkFunction = bulkFunction;
             this.failureConsumer = failureConsumer;
@@ -180,36 +201,39 @@ public class TransformIndexerTests extends ESTestCase {
             fail("failIndexer should not be called, received error: " + message);
         }
 
-        @Override
-        protected void sourceHasChanged(ActionListener<Boolean> listener) {
-            listener.onResponse(false);
-        }
-
     }
 
     @Before
     public void setUpMocks() {
-        client = mock(Client.class);
+        client = new NoOpClient(getTestName());
-        ThreadPool threadPool = mock(ThreadPool.class);
+    }
-        when(client.threadPool()).thenReturn(threadPool);
+
-        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+    @After
+    public void tearDownClient() {
+        client.close();
     }
 
     public void testPageSizeAdapt() throws Exception {
         Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000);
-        TransformConfig config = new TransformConfig(randomAlphaOfLength(10),
+        TransformConfig config = new TransformConfig(
+            randomAlphaOfLength(10),
             randomSourceConfig(),
             randomDestConfig(),
             null,
             null,
             null,
             new PivotConfig(GroupConfigTests.randomGroupConfig(), AggregationConfigTests.randomAggregationConfig(), pageSize),
-            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)
+        );
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final long initialPageSize = pageSize == null ? Pivot.DEFAULT_INITIAL_PAGE_SIZE : pageSize;
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> {
-            throw new SearchPhaseExecutionException("query", "Partial shards failure", new ShardSearchFailure[] {
+            throw new SearchPhaseExecutionException(
-                    new ShardSearchFailure(new CircuitBreakingException("to much memory", 110, 100, Durability.TRANSIENT)) });
+                "query",
+                "Partial shards failure",
+                new ShardSearchFailure[] {
+                    new ShardSearchFailure(new CircuitBreakingException("to much memory", 110, 100, Durability.TRANSIENT)) }
+            );
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100);
@@ -224,9 +248,24 @@ public class TransformIndexerTests extends ESTestCase {
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             TransformAuditor auditor = new TransformAuditor(client, "node_1");
+            TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class));
 
-            MockedTransformIndexer indexer = new MockedTransformIndexer(executor, config, Collections.emptyMap(), auditor, state, null,
+            MockedTransformIndexer indexer = new MockedTransformIndexer(
-                    new TransformIndexerStats(), searchFunction, bulkFunction, failureConsumer);
+                executor,
+                mock(TransformConfigManager.class),
+                mock(CheckpointProvider.class),
+                new TransformProgressGatherer(client),
+                config,
+                Collections.emptyMap(),
+                auditor,
+                state,
+                null,
+                new TransformIndexerStats(),
+                context,
+                searchFunction,
+                bulkFunction,
+                failureConsumer
+            );
             final CountDownLatch latch = indexer.newLatch(1);
             indexer.start();
             assertThat(indexer.getState(), equalTo(IndexerState.STARTED));
@@ -234,7 +273,7 @@ public class TransformIndexerTests extends ESTestCase {
             assertThat(indexer.getState(), equalTo(IndexerState.INDEXING));
 
             latch.countDown();
-            assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED)));
+            assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED)), 10, TimeUnit.MINUTES);
             long pageSizeAfterFirstReduction = indexer.getPageSize();
             assertThat(initialPageSize, greaterThan(pageSizeAfterFirstReduction));
             assertThat(pageSizeAfterFirstReduction, greaterThan((long) TransformIndexer.MINIMUM_PAGE_SIZE));
@@ -261,22 +300,35 @@ public class TransformIndexerTests extends ESTestCase {
 
     public void testDoProcessAggNullCheck() {
         Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000);
-        TransformConfig config = new TransformConfig(randomAlphaOfLength(10),
+        TransformConfig config = new TransformConfig(
+            randomAlphaOfLength(10),
             randomSourceConfig(),
             randomDestConfig(),
             null,
             null,
             null,
             new PivotConfig(GroupConfigTests.randomGroupConfig(), AggregationConfigTests.randomAggregationConfig(), pageSize),
-            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)
-        SearchResponse searchResponse = new SearchResponse(new InternalSearchResponse(
+        );
-            new SearchHits(
+        SearchResponse searchResponse = new SearchResponse(
-                new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f),
+            new InternalSearchResponse(
+                new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f),
                 // Simulate completely null aggs
                 null,
                 new Suggest(Collections.emptyList()),
-            new SearchProfileShardResults(Collections.emptyMap()), false, false, 1),
+                new SearchProfileShardResults(Collections.emptyMap()),
-            "", 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                false,
+                false,
+                1
+            ),
+            "",
+            1,
+            1,
+            0,
+            0,
+            ShardSearchFailure.EMPTY_ARRAY,
+            SearchResponse.Clusters.EMPTY
+        );
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> searchResponse;
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100);
@@ -291,9 +343,24 @@ public class TransformIndexerTests extends ESTestCase {
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         try {
             TransformAuditor auditor = mock(TransformAuditor.class);
+            TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class));
 
-            MockedTransformIndexer indexer = new MockedTransformIndexer(executor, config, Collections.emptyMap(), auditor, state, null,
+            MockedTransformIndexer indexer = new MockedTransformIndexer(
-                new TransformIndexerStats(), searchFunction, bulkFunction, failureConsumer);
+                executor,
+                mock(TransformConfigManager.class),
+                mock(CheckpointProvider.class),
+                new TransformProgressGatherer(client),
+                config,
+                Collections.emptyMap(),
+                auditor,
+                state,
+                null,
+                new TransformIndexerStats(),
+                context,
+                searchFunction,
+                bulkFunction,
+                failureConsumer
+            );
 
             IterationResult<TransformIndexerPosition> newPosition = indexer.doProcess(searchResponse);
             assertThat(newPosition.getToIndex(), is(empty()));

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java

@@ -32,6 +32,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
 import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
+import org.elasticsearch.xpack.transform.Transform;
 import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
 import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
@@ -53,48 +54,71 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase {
         MetaData.Builder metaData = MetaData.builder();
         RoutingTable.Builder routingTable = RoutingTable.builder();
         addIndices(metaData, routingTable);
-        PersistentTasksCustomMetaData.Builder pTasksBuilder = PersistentTasksCustomMetaData.builder()
+        PersistentTasksCustomMetaData.Builder pTasksBuilder = PersistentTasksCustomMetaData
-            .addTask("transform-task-1",
+            .builder()
+            .addTask(
+                "transform-task-1",
                 TransformTaskParams.NAME,
                 new TransformTaskParams("transform-task-1", Version.CURRENT, null),
-                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-1-tasks", ""))
+                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-1-tasks", "")
-            .addTask("transform-task-2",
+            )
+            .addTask(
+                "transform-task-2",
                 TransformTaskParams.NAME,
                 new TransformTaskParams("transform-task-2", Version.CURRENT, null),
-                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-2-tasks", ""))
+                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-2-tasks", "")
-            .addTask("transform-task-3",
+            )
+            .addTask(
+                "transform-task-3",
                 TransformTaskParams.NAME,
                 new TransformTaskParams("transform-task-3", Version.CURRENT, null),
-                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-2-tasks", ""));
+                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-2-tasks", "")
+            );
 
         PersistentTasksCustomMetaData pTasks = pTasksBuilder.build();
 
         metaData.putCustom(PersistentTasksCustomMetaData.TYPE, pTasks);
 
-        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder()
+        DiscoveryNodes.Builder nodes = DiscoveryNodes
-            .add(new DiscoveryNode("past-data-node-1",
+            .builder()
+            .add(
+                new DiscoveryNode(
+                    "past-data-node-1",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
-                Version.V_7_4_0))
+                    Version.V_7_4_0
-            .add(new DiscoveryNode("current-data-node-with-2-tasks",
+                )
+            )
+            .add(
+                new DiscoveryNode(
+                    "current-data-node-with-2-tasks",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
-                Version.CURRENT))
+                    Version.CURRENT
-            .add(new DiscoveryNode("non-data-node-1",
+                )
+            )
+            .add(
+                new DiscoveryNode(
+                    "non-data-node-1",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     Collections.singleton(DiscoveryNodeRole.MASTER_ROLE),
-                Version.CURRENT))
+                    Version.CURRENT
-            .add(new DiscoveryNode("current-data-node-with-1-tasks",
+                )
+            )
+            .add(
+                new DiscoveryNode(
+                    "current-data-node-with-1-tasks",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
-                Version.CURRENT));
+                    Version.CURRENT
+                )
+            );
 
-        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"))
+        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")).nodes(nodes);
-            .nodes(nodes);
         csBuilder.routingTable(routingTable.build());
         csBuilder.metaData(metaData);
 
@@ -102,60 +126,80 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase {
         Client client = mock(Client.class);
         TransformAuditor mockAuditor = mock(TransformAuditor.class);
         TransformConfigManager transformsConfigManager = new TransformConfigManager(client, xContentRegistry());
-        TransformCheckpointService transformCheckpointService = new TransformCheckpointService(client,
+        TransformCheckpointService transformCheckpointService = new TransformCheckpointService(
-            transformsConfigManager, mockAuditor);
+            client,
-        ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY,
+            transformsConfigManager,
-            Collections.singleton(TransformTask.NUM_FAILURE_RETRIES_SETTING));
+            mockAuditor
+        );
+        ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY, Collections.singleton(Transform.NUM_FAILURE_RETRIES_SETTING));
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.getClusterSettings()).thenReturn(cSettings);
         when(clusterService.state()).thenReturn(TransformInternalIndexTests.STATE_WITH_LATEST_VERSIONED_INDEX_TEMPLATE);
-        TransformPersistentTasksExecutor executor = new TransformPersistentTasksExecutor(client,
+        TransformPersistentTasksExecutor executor = new TransformPersistentTasksExecutor(
+            client,
             transformsConfigManager,
-            transformCheckpointService, mock(SchedulerEngine.class),
+            transformCheckpointService,
+            mock(SchedulerEngine.class),
             new TransformAuditor(client, ""),
             mock(ThreadPool.class),
             clusterService,
-            Settings.EMPTY);
+            Settings.EMPTY
+        );
 
-        assertThat(executor.getAssignment(new TransformTaskParams("new-task-id", Version.CURRENT, null), cs).getExecutorNode(),
+        assertThat(
-            equalTo("current-data-node-with-1-tasks"));
+            executor.getAssignment(new TransformTaskParams("new-task-id", Version.CURRENT, null), cs).getExecutorNode(),
-        assertThat(executor.getAssignment(new TransformTaskParams("new-old-task-id", Version.V_7_2_0, null), cs).getExecutorNode(),
+            equalTo("current-data-node-with-1-tasks")
-            equalTo("past-data-node-1"));
+        );
     }
 
     public void testDoNotSelectOldNodes() {
         MetaData.Builder metaData = MetaData.builder();
         RoutingTable.Builder routingTable = RoutingTable.builder();
         addIndices(metaData, routingTable);
-        PersistentTasksCustomMetaData.Builder pTasksBuilder = PersistentTasksCustomMetaData.builder()
+        PersistentTasksCustomMetaData.Builder pTasksBuilder = PersistentTasksCustomMetaData
-            .addTask("transform-task-1",
+            .builder()
+            .addTask(
+                "transform-task-1",
                 TransformTaskParams.NAME,
                 new TransformTaskParams("transform-task-1", Version.CURRENT, null),
-                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-1-task", ""));
+                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-1-task", "")
+            );
 
         PersistentTasksCustomMetaData pTasks = pTasksBuilder.build();
 
         metaData.putCustom(PersistentTasksCustomMetaData.TYPE, pTasks);
 
-        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder()
+        DiscoveryNodes.Builder nodes = DiscoveryNodes
-            .add(new DiscoveryNode("old-data-node-1",
+            .builder()
+            .add(
+                new DiscoveryNode(
+                    "old-data-node-1",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
-                Version.V_7_2_0))
+                    Version.V_7_2_0
-            .add(new DiscoveryNode("current-data-node-with-1-task",
+                )
+            )
+            .add(
+                new DiscoveryNode(
+                    "current-data-node-with-1-task",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
-                Version.CURRENT))
+                    Version.CURRENT
-            .add(new DiscoveryNode("non-data-node-1",
+                )
+            )
+            .add(
+                new DiscoveryNode(
+                    "non-data-node-1",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     Collections.singleton(DiscoveryNodeRole.MASTER_ROLE),
-                Version.CURRENT));
+                    Version.CURRENT
+                )
+            );
 
-        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"))
+        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")).nodes(nodes);
-            .nodes(nodes);
         csBuilder.routingTable(routingTable.build());
         csBuilder.metaData(metaData);
 
@@ -163,41 +207,57 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase {
         Client client = mock(Client.class);
         TransformAuditor mockAuditor = mock(TransformAuditor.class);
         TransformConfigManager transformsConfigManager = new TransformConfigManager(client, xContentRegistry());
-        TransformCheckpointService transformCheckpointService = new TransformCheckpointService(client,
+        TransformCheckpointService transformCheckpointService = new TransformCheckpointService(
-            transformsConfigManager, mockAuditor);
+            client,
-        ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY,
+            transformsConfigManager,
-            Collections.singleton(TransformTask.NUM_FAILURE_RETRIES_SETTING));
+            mockAuditor
+        );
+        ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY, Collections.singleton(Transform.NUM_FAILURE_RETRIES_SETTING));
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.getClusterSettings()).thenReturn(cSettings);
         when(clusterService.state()).thenReturn(TransformInternalIndexTests.STATE_WITH_LATEST_VERSIONED_INDEX_TEMPLATE);
-        TransformPersistentTasksExecutor executor = new TransformPersistentTasksExecutor(client,
+        TransformPersistentTasksExecutor executor = new TransformPersistentTasksExecutor(
+            client,
             transformsConfigManager,
-            transformCheckpointService, mock(SchedulerEngine.class),
+            transformCheckpointService,
+            mock(SchedulerEngine.class),
             new TransformAuditor(client, ""),
             mock(ThreadPool.class),
             clusterService,
-            Settings.EMPTY);
+            Settings.EMPTY
+        );
 
         // old-data-node-1 prevents assignment
         assertNull(executor.getAssignment(new TransformTaskParams("new-task-id", Version.CURRENT, null), cs).getExecutorNode());
 
         // remove the old 7.2 node
-        nodes = DiscoveryNodes.builder()
+        nodes = DiscoveryNodes
-            .add(new DiscoveryNode("current-data-node-with-1-task",
+            .builder()
+            .add(
+                new DiscoveryNode(
+                    "current-data-node-with-1-task",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
-                Version.CURRENT))
+                    Version.CURRENT
-            .add(new DiscoveryNode("non-data-node-1",
+                )
+            )
+            .add(
+                new DiscoveryNode(
+                    "non-data-node-1",
                     buildNewFakeTransportAddress(),
                     Collections.emptyMap(),
                     Collections.singleton(DiscoveryNodeRole.MASTER_ROLE),
-                Version.CURRENT));
+                    Version.CURRENT
+                )
+            );
         csBuilder.nodes(nodes);
         cs = csBuilder.build();
 
-        assertThat(executor.getAssignment(new TransformTaskParams("new-old-task-id", Version.V_7_2_0, null), cs).getExecutorNode(),
+        assertThat(
-            equalTo("current-data-node-with-1-task"));
+            executor.getAssignment(new TransformTaskParams("new-old-task-id", Version.V_7_2_0, null), cs).getExecutorNode(),
+            equalTo("current-data-node-with-1-task")
+        );
     }
 
     public void testVerifyIndicesPrimaryShardsAreActive() {
@@ -220,11 +280,20 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase {
         } else {
             Index index = new Index(indexToRemove, "_uuid");
             ShardId shardId = new ShardId(index, 0);
-            ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+            ShardRouting shardRouting = ShardRouting
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""));
+                .newUnassigned(
+                    shardId,
+                    true,
+                    RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
-            routingTable.add(IndexRoutingTable.builder(index)
+            routingTable
-                .addIndexShard(new IndexShardRoutingTable.Builder(shardId).addShard(shardRouting).build()));
+                .add(
+                    IndexRoutingTable
+                        .builder(index)
+                        .addIndexShard(new IndexShardRoutingTable.Builder(shardId).addShard(shardRouting).build())
+                );
         }
 
         csBuilder.routingTable(routingTable.build());
@@ -240,7 +309,10 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase {
         indices.add(TransformInternalIndexConstants.LATEST_INDEX_NAME);
         for (String indexName : indices) {
             IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName);
-            indexMetaData.settings(Settings.builder()
+            indexMetaData
+                .settings(
+                    Settings
+                        .builder()
                         .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
@@ -248,12 +320,21 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase {
             metaData.put(indexMetaData);
             Index index = new Index(indexName, "_uuid");
             ShardId shardId = new ShardId(index, 0);
-            ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+            ShardRouting shardRouting = ShardRouting
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""));
+                .newUnassigned(
+                    shardId,
+                    true,
+                    RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
             shardRouting = shardRouting.moveToStarted();
-            routingTable.add(IndexRoutingTable.builder(index)
+            routingTable
-                .addIndexShard(new IndexShardRoutingTable.Builder(shardId).addShard(shardRouting).build()));
+                .add(
+                    IndexRoutingTable
+                        .builder(index)
+                        .addIndexShard(new IndexShardRoutingTable.Builder(shardId).addShard(shardRouting).build())
+                );
         }
     }