NIFI-1492: - Limiting the amount of state entries returned to a client. - Code clean up.

Signed-off-by: joewitt <joewitt@apache.org>

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/StateMapDTO.java

@@ -28,6 +28,7 @@ import java.util.List;
 public class StateMapDTO {
 
     private String scope;
+    private int totalEntryCount;
     private List<StateEntryDTO> state;
 
     /**
@@ -44,6 +45,20 @@ public class StateMapDTO {
         this.scope = scope;
     }
 
+    /**
+     * @return The total number of state entries. When the state map is lengthy, only of portion of the entries are returned.
+     */
+    @ApiModelProperty(
+        value = "The total number of state entries. When the state map is lengthy, only of portion of the entries are returned."
+    )
+    public int getTotalEntryCount() {
+        return totalEntryCount;
+    }
+
+    public void setTotalEntryCount(int totalEntryCount) {
+        this.totalEntryCount = totalEntryCount;
+    }
+
     /**
      * @return The state
      */

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/impl/WebClusterManager.java

@@ -151,6 +151,7 @@ import org.apache.nifi.controller.service.ControllerServiceNode;
 import org.apache.nifi.controller.service.ControllerServiceProvider;
 import org.apache.nifi.controller.service.ControllerServiceState;
 import org.apache.nifi.controller.service.StandardControllerServiceProvider;
+import org.apache.nifi.controller.state.SortedStateUtils;
 import org.apache.nifi.controller.state.manager.StandardStateManagerProvider;
 import org.apache.nifi.controller.status.ProcessGroupStatus;
 import org.apache.nifi.controller.status.RemoteProcessGroupStatus;
@@ -2574,8 +2575,9 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
     }
 
     private void mergeComponentState(final ComponentStateDTO componentState, Map<NodeIdentifier, ComponentStateDTO> componentStateMap) {
-        final List<StateEntryDTO> localStateEntries = new ArrayList<>();
+        List<StateEntryDTO> localStateEntries = new ArrayList<>();
 
+        int totalStateEntries = 0;
         for (final Map.Entry<NodeIdentifier, ComponentStateDTO> nodeEntry : componentStateMap.entrySet()) {
             final ComponentStateDTO nodeComponentState = nodeEntry.getValue();
             final NodeIdentifier nodeId = nodeEntry.getKey();
@@ -2583,6 +2585,8 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
 
             final StateMapDTO nodeLocalStateMap = nodeComponentState.getLocalState();
             if (nodeLocalStateMap.getState() != null) {
+                totalStateEntries += nodeLocalStateMap.getTotalEntryCount();
+
                 for (final StateEntryDTO nodeStateEntry : nodeLocalStateMap.getState()) {
                     nodeStateEntry.setClusterNodeId(nodeId.getId());
                     nodeStateEntry.setClusterNodeAddress(nodeAddress);
@@ -2591,7 +2595,16 @@ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, C
             }
         }
 
+        // ensure appropriate sort
+        Collections.sort(localStateEntries, SortedStateUtils.getEntryDtoComparator());
+
+        // sublist if necessary
+        if (localStateEntries.size() > SortedStateUtils.MAX_COMPONENT_STATE_ENTRIES) {
+            localStateEntries = localStateEntries.subList(0, SortedStateUtils.MAX_COMPONENT_STATE_ENTRIES);
+        }
+
         // add all the local state entries
+        componentState.getLocalState().setTotalEntryCount(totalStateEntries);
         componentState.getLocalState().setState(localStateEntries);
     }
 

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/state/SortedStateUtils.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.state;
+
+import org.apache.nifi.web.api.dto.StateEntryDTO;
+
+import java.text.Collator;
+import java.util.Comparator;
+import java.util.Locale;
+
+public class SortedStateUtils {
+
+    /**
+     * The maximum number of state entries to return to a client
+     */
+    public static final int MAX_COMPONENT_STATE_ENTRIES = 500;
+
+    /**
+     * Gets a comparator for comparing state entry keys.
+     *
+     * @return comparator for comparing state entry keys
+     */
+    public static Comparator<String> getKeyComparator() {
+        final Collator collator = Collator.getInstance(Locale.US);
+        return new Comparator<String>() {
+            @Override
+            public int compare(String s1, String s2) {
+                return collator.compare(s1, s2);
+            }
+        };
+    }
+
+    /**
+     * Gets a comparator for comparing state entry keys.
+     *
+     * @return comparator for comparing state entry keys
+     */
+    public static Comparator<StateEntryDTO> getEntryDtoComparator() {
+        final Collator collator = Collator.getInstance(Locale.US);
+        return new Comparator<StateEntryDTO>() {
+            @Override
+            public int compare(StateEntryDTO o1, StateEntryDTO o2) {
+                return collator.compare(o1.getKey(), o2.getKey());
+            }
+        };
+    }
+}

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java

@@ -71,6 +71,7 @@ import org.apache.nifi.controller.repository.claim.ContentClaim;
 import org.apache.nifi.controller.repository.claim.ResourceClaim;
 import org.apache.nifi.controller.service.ControllerServiceNode;
 import org.apache.nifi.controller.service.ControllerServiceReference;
+import org.apache.nifi.controller.state.SortedStateUtils;
 import org.apache.nifi.controller.status.ConnectionStatus;
 import org.apache.nifi.controller.status.PortStatus;
 import org.apache.nifi.controller.status.ProcessGroupStatus;
@@ -137,11 +138,13 @@ import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -319,14 +322,21 @@ public final class DtoFactory {
         final StateMapDTO dto = new StateMapDTO();
         dto.setScope(scope.toString());
 
-        final List<StateEntryDTO> stateEntries = new ArrayList<>();
+        final TreeMap<String, String> sortedState = new TreeMap(SortedStateUtils.getKeyComparator());
         final Map<String, String> state = stateMap.toMap();
-        for (final Map.Entry<String, String> entry : state.entrySet()) {
+        sortedState.putAll(state);
+
+        int count = 0;
+        final List<StateEntryDTO> stateEntries = new ArrayList<>();
+        final Set<Map.Entry<String, String>> entrySet = sortedState.entrySet();
+        for (final Iterator<Entry<String, String>> iter = entrySet.iterator(); iter.hasNext() && count++ < SortedStateUtils.MAX_COMPONENT_STATE_ENTRIES;) {
+            final Map.Entry<String, String> entry = iter.next();
             final StateEntryDTO entryDTO = new StateEntryDTO();
             entryDTO.setKey(entry.getKey());
             entryDTO.setValue(entry.getValue());
             stateEntries.add(entryDTO);
         }
+        dto.setTotalEntryCount(state.size());
         dto.setState(stateEntries);
 
         return dto;

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/component-state-dialog.jsp

@@ -29,6 +29,10 @@
                 <div id="component-state-description" class="ellipsis multiline"></div>
             </div>
         </div>
+        <div>
+            <div id="component-state-partial-results-container" class="hidden">
+                Showing partial results
+            </div>
             <div id="component-state-filter-controls">
                 <div id="component-state-filter-container">
                     <input type="text" id="component-state-filter"/>
@@ -37,6 +41,8 @@
                     Displaying&nbsp;<span id="displayed-component-state-entries"></span>&nbsp;of&nbsp;<span id="total-component-state-entries"></span>
                 </div>
             </div>
+            <div class="clear"></div>
+        </div>
         <div id="component-state-table"></div>
         <div id="clear-link-container">
             <span id="clear-link" class="link">Clear state</span>

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/component-state.css

@@ -39,6 +39,15 @@
     Component state filter
 */
 
+#component-state-partial-results-container {
+    float: left;
+    color: #9f6000;
+    font-size: 9px;
+    font-weight: bold;
+    line-height: normal;
+    margin-top: 34px;
+}
+
 #component-state-filter-controls {
     float: right;
     margin-top: 10px;

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-component-state.js

@@ -121,6 +121,9 @@ nf.ComponentState = (function () {
         var componentStateData = componentStateGrid.getData();
         componentStateData.setItems([]);
 
+        // hide the partial results details message
+        $('#component-state-partial-results-container').hide();
+
         // clear the total number entries
         $('#displayed-component-state-entries').text('0');
         $('#total-component-state-entries').text('0');
@@ -133,6 +136,8 @@ nf.ComponentState = (function () {
      */
     var loadComponentState = function (localState, clusterState) {
         var count = 0;
+        var totalEntries = 0;
+        var showPartialDetails = false;
 
         var componentStateGrid = $('#component-state-table').data('gridInstance');
         var componentStateData = componentStateGrid.getData();
@@ -148,6 +153,11 @@ nf.ComponentState = (function () {
                     scope: stateEntry.clusterNodeAddress
                 }, stateEntry));
             });
+            totalEntries += localState.totalEntryCount;
+
+            if (nf.Common.isDefinedAndNotNull(localState.state) && localState.totalEntryCount !== localState.state.length) {
+                showPartialDetails = true;
+            }
         }
 
         if (nf.Common.isDefinedAndNotNull(clusterState)) {
@@ -157,14 +167,23 @@ nf.ComponentState = (function () {
                     scope: 'Cluster'
                 }, stateEntry));
             });
+            totalEntries += clusterState.totalEntryCount;
+
+            if (nf.Common.isDefinedAndNotNull(clusterState.state) && clusterState.totalEntryCount !== clusterState.state.length) {
+                showPartialDetails = true;
+            }
         }
 
         // complete the update
         componentStateData.endUpdate();
         componentStateData.reSort();
 
+        if (showPartialDetails) {
+            $('#component-state-partial-results-container').show();
+        }
+
         // update the total number of state entries
-        $('#total-component-state-entries').text(count);
+        $('#total-component-state-entries').text(nf.Common.formatInteger(totalEntries));
     };
 
     /**

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor-configuration.js

@@ -590,15 +590,13 @@ nf.ProcessorConfiguration = (function () {
                 }
                 
                 // once everything is loaded, show the dialog
-                $.when.apply(window, requests).done(function (processorResponse, historyResponse, stateResponse) {
+                $.when.apply(window, requests).done(function (processorResponse, historyResponse) {
                     // get the updated processor
                     processor = processorResponse[0].processor;
                     
                     // get the processor history
                     var processorHistory = historyResponse[0].componentHistory;
 
-                    console.log(stateResponse);
-
                     // record the processor details
                     $('#processor-configuration').data('processorDetails', processor);