From 5957f6ee95052a2d97377f422900087dade67470 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Fri, 1 Sep 2017 11:16:29 +0200 Subject: [PATCH] NIFI-4341 - add provenance repository storage usage in UI - addressing review comments - This closes #2119 --- .../org/apache/nifi/util/file/FileUtils.java | 18 +++ .../nifi/provenance/ProvenanceRepository.java | 27 ++++ .../provenance/MockProvenanceRepository.java | 17 +++ .../api/dto/SystemDiagnosticsSnapshotDTO.java | 30 ++++- .../nifi/cluster/manager/StatusMerger.java | 1 + .../nifi/controller/FlowController.java | 2 +- .../repository/FileSystemRepository.java | 12 +- .../nifi/diagnostics/SystemDiagnostics.java | 16 +++ .../diagnostics/SystemDiagnosticsFactory.java | 29 ++++- .../apache/nifi/web/api/dto/DtoFactory.java | 7 ++ .../partials/cluster/cluster-content.jsp | 3 + .../summary/system-diagnostics-dialog.jsp | 110 ++++++++-------- .../src/main/webapp/css/summary.css | 17 ++- .../webapp/js/nf/cluster/nf-cluster-table.js | 118 +++++++++++++++++- .../webapp/js/nf/summary/nf-summary-table.js | 13 ++ .../PersistentProvenanceRepository.java | 35 ++++++ .../WriteAheadProvenanceRepository.java | 47 ++++++- .../VolatileProvenanceRepository.java | 24 +++- 18 files changed, 453 insertions(+), 73 deletions(-) diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java index 960bc401a8..13b6a4b0ce 100644 --- a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java +++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java @@ -592,4 +592,22 @@ public class FileUtils { return digest.digest(); } + /** + * Returns the capacity for a given path + * @param path path + * @return total space + */ + public static long getContainerCapacity(final Path path) { + return path.toFile().getTotalSpace(); + } + + /** + * Returns the free capacity for a given path + * @param path path + * @return usable space + */ + public static long getContainerUsableSpace(final Path path) { + return path.toFile().getUsableSpace(); + } + } diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/provenance/ProvenanceRepository.java b/nifi-framework-api/src/main/java/org/apache/nifi/provenance/ProvenanceRepository.java index 516a36defd..6d15d85bf4 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/provenance/ProvenanceRepository.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/provenance/ProvenanceRepository.java @@ -27,6 +27,7 @@ import org.apache.nifi.provenance.search.SearchableField; import java.io.IOException; import java.util.List; +import java.util.Set; public interface ProvenanceRepository extends ProvenanceEventRepository { @@ -181,4 +182,30 @@ public interface ProvenanceRepository extends ProvenanceEventRepository { * {@link ProvenanceRepository#submitQuery(Query, NiFiUser)} method */ List getSearchableAttributes(); + + /** + * @return the names of all Containers that exist for this Provenance + * Repository + */ + Set getContainerNames(); + + /** + * @param containerName name of container to check capacity on + * @return the maximum number of bytes that can be stored in the storage + * mechanism that backs the container with the given name + * @throws java.io.IOException if unable to check capacity + * @throws IllegalArgumentException if no container exists with the given + * name + */ + long getContainerCapacity(String containerName) throws IOException; + + /** + * @param containerName to check space on + * @return the number of bytes available to be used used by the storage + * mechanism that backs the container with the given name + * @throws java.io.IOException if unable to check space + * @throws IllegalArgumentException if no container exists with the given + * name + */ + long getContainerUsableSpace(String containerName) throws IOException; } diff --git a/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceRepository.java b/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceRepository.java index 53c3c2e62f..0b4afcd023 100644 --- a/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceRepository.java +++ b/nifi-mock/src/main/java/org/apache/nifi/provenance/MockProvenanceRepository.java @@ -19,7 +19,9 @@ package org.apache.nifi.provenance; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.apache.nifi.authorization.Authorizer; @@ -151,4 +153,19 @@ public class MockProvenanceRepository implements ProvenanceRepository { public ProvenanceEventRepository getProvenanceEventRepository() { return this; } + + @Override + public long getContainerCapacity(String containerName) throws IOException { + return 0; + } + + @Override + public Set getContainerNames() { + return new HashSet(); + } + + @Override + public long getContainerUsableSpace(String containerName) throws IOException { + return 0; + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/SystemDiagnosticsSnapshotDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/SystemDiagnosticsSnapshotDTO.java index 47c58f6da6..294977674d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/SystemDiagnosticsSnapshotDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/SystemDiagnosticsSnapshotDTO.java @@ -62,6 +62,7 @@ public class SystemDiagnosticsSnapshotDTO implements Cloneable { private StorageUsageDTO flowFileRepositoryStorageUsage; private Set contentRepositoryStorageUsage; + private Set provenanceRepositoryStorageUsage; private Set garbageCollection; private Date statsLastRefreshed; @@ -204,6 +205,15 @@ public class SystemDiagnosticsSnapshotDTO implements Cloneable { this.contentRepositoryStorageUsage = contentRepositoryStorageUsage; } + @ApiModelProperty("The provenance repository storage usage.") + public Set getProvenanceRepositoryStorageUsage() { + return provenanceRepositoryStorageUsage; + } + + public void setProvenanceRepositoryStorageUsage(Set provenanceRepositoryStorageUsage) { + this.provenanceRepositoryStorageUsage = provenanceRepositoryStorageUsage; + } + @ApiModelProperty("The flowfile repository storage usage.") public StorageUsageDTO getFlowFileRepositoryStorageUsage() { return flowFileRepositoryStorageUsage; @@ -357,14 +367,26 @@ public class SystemDiagnosticsSnapshotDTO implements Cloneable { final Set contentRepoStorageUsage = new LinkedHashSet<>(); other.setContentRepositoryStorageUsage(contentRepoStorageUsage); - for (final StorageUsageDTO usage : getContentRepositoryStorageUsage()) { - contentRepoStorageUsage.add(usage.clone()); + if (getContentRepositoryStorageUsage() != null) { + for (final StorageUsageDTO usage : getContentRepositoryStorageUsage()) { + contentRepoStorageUsage.add(usage.clone()); + } + } + + final Set provenanceRepoStorageUsage = new LinkedHashSet<>(); + other.setProvenanceRepositoryStorageUsage(provenanceRepoStorageUsage); + if (getProvenanceRepositoryStorageUsage() != null) { + for (final StorageUsageDTO usage : getProvenanceRepositoryStorageUsage()) { + provenanceRepoStorageUsage.add(usage.clone()); + } } final Set gcUsage = new LinkedHashSet<>(); other.setGarbageCollection(gcUsage); - for (final GarbageCollectionDTO gcDto : getGarbageCollection()) { - gcUsage.add(gcDto.clone()); + if (getGarbageCollection() != null) { + for (final GarbageCollectionDTO gcDto : getGarbageCollection()) { + gcUsage.add(gcDto.clone()); + } } other.setVersionInfo(getVersionInfo().clone()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java index 9cceaf7c08..962ab2cfc7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java @@ -608,6 +608,7 @@ public class StatusMerger { target.setUsedNonHeapBytes(target.getUsedNonHeapBytes() + toMerge.getUsedNonHeapBytes()); merge(target.getContentRepositoryStorageUsage(), toMerge.getContentRepositoryStorageUsage()); + merge(target.getProvenanceRepositoryStorageUsage(), toMerge.getProvenanceRepositoryStorageUsage()); merge(target.getFlowFileRepositoryStorageUsage(), toMerge.getFlowFileRepositoryStorageUsage()); mergeGarbageCollection(target.getGarbageCollection(), toMerge.getGarbageCollection()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 94645cf827..9c181ffaf1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -1684,7 +1684,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R public SystemDiagnostics getSystemDiagnostics() { final SystemDiagnosticsFactory factory = new SystemDiagnosticsFactory(); - return factory.create(flowFileRepository, contentRepository); + return factory.create(flowFileRepository, contentRepository, provenanceRepository); } // diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java index 0bfe6a18fa..6a8d314aa8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java @@ -404,12 +404,16 @@ public class FileSystemRepository implements ContentRepository { @Override public long getContainerCapacity(final String containerName) throws IOException { final Path path = containers.get(containerName); + if (path == null) { throw new IllegalArgumentException("No container exists with name " + containerName); } - long capacity = path.toFile().getTotalSpace(); + + long capacity = FileUtils.getContainerCapacity(path); + if(capacity==0) { - throw new IOException("System returned total space of the partition for " + containerName + " is zero byte. Nifi can not create a zero sized FileSystemRepository"); + throw new IOException("System returned total space of the partition for " + containerName + " is zero byte. " + + "Nifi can not create a zero sized FileSystemRepository."); } return capacity; @@ -418,10 +422,12 @@ public class FileSystemRepository implements ContentRepository { @Override public long getContainerUsableSpace(String containerName) throws IOException { final Path path = containers.get(containerName); + if (path == null) { throw new IllegalArgumentException("No container exists with name " + containerName); } - return path.toFile().getUsableSpace(); + + return FileUtils.getContainerUsableSpace(path); } @Override diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java index e599e1f572..b5d1597190 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java @@ -43,6 +43,7 @@ public class SystemDiagnostics implements Cloneable { private StorageUsage flowFileRepositoryStorageUsage; private Map contentRepositoryStorageUsage; + private Map provenanceRepositoryStorageUsage; private Map garbageCollection; private long creationTimestamp; @@ -95,6 +96,10 @@ public class SystemDiagnostics implements Cloneable { this.contentRepositoryStorageUsage = contentRepositoryStorageUsage; } + public void setProvenanceRepositoryStorageUsage(final Map provenanceRepositoryStorageUsage) { + this.provenanceRepositoryStorageUsage = provenanceRepositoryStorageUsage; + } + public long getTotalNonHeap() { return totalNonHeap; } @@ -143,6 +148,10 @@ public class SystemDiagnostics implements Cloneable { return contentRepositoryStorageUsage; } + public Map getProvenanceRepositoryStorageUsage() { + return provenanceRepositoryStorageUsage; + } + public long getFreeNonHeap() { return totalNonHeap - usedNonHeap; } @@ -206,6 +215,13 @@ public class SystemDiagnostics implements Cloneable { clonedMap.put(entry.getKey(), entry.getValue().clone()); } } + if(provenanceRepositoryStorageUsage != null) { + final Map clonedMap = new LinkedHashMap<>(); + clonedObj.setProvenanceRepositoryStorageUsage(clonedMap); + for (final Map.Entry entry : provenanceRepositoryStorageUsage.entrySet()) { + clonedMap.put(entry.getKey(), entry.getValue().clone()); + } + } if (garbageCollection != null) { final Map clonedMap = new LinkedHashMap<>(); clonedObj.setGarbageCollection(clonedMap); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnosticsFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnosticsFactory.java index 86a772ab04..45a8af740e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnosticsFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnosticsFactory.java @@ -32,7 +32,7 @@ import java.util.Set; import org.apache.nifi.controller.repository.ContentRepository; import org.apache.nifi.controller.repository.FlowFileRepository; - +import org.apache.nifi.provenance.ProvenanceRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +44,7 @@ public class SystemDiagnosticsFactory { private final Logger logger = LoggerFactory.getLogger(SystemDiagnosticsFactory.class); - public SystemDiagnostics create(final FlowFileRepository flowFileRepo, final ContentRepository contentRepo) { + public SystemDiagnostics create(final FlowFileRepository flowFileRepo, final ContentRepository contentRepo, ProvenanceRepository provenanceRepository) { final SystemDiagnostics systemDiagnostics = new SystemDiagnostics(); final MemoryMXBean memory = ManagementFactory.getMemoryMXBean(); @@ -119,6 +119,31 @@ public class SystemDiagnosticsFactory { } systemDiagnostics.setContentRepositoryStorageUsage(fileRepositoryUsage); + // get provenance repository disk usage + final Set provContainerNames = provenanceRepository.getContainerNames(); + final Map provRepositoryUsage = new LinkedHashMap<>(provContainerNames.size()); + for (final String containerName : provContainerNames) { + long containerCapacity = -1L; + long containerFree = 0L; + + try { + containerFree = provenanceRepository.getContainerUsableSpace(containerName); + containerCapacity = provenanceRepository.getContainerCapacity(containerName); + } catch (final IOException ioe) { + logger.warn("Unable to determine Provenance Repository usage for container {} due to {}", containerName, ioe.toString()); + if (logger.isDebugEnabled()) { + logger.warn("", ioe); + } + } + + final StorageUsage storageUsage = new StorageUsage(); + storageUsage.setIdentifier(containerName); + storageUsage.setFreeSpace(containerFree); + storageUsage.setTotalSpace(containerCapacity); + provRepositoryUsage.put(containerName, storageUsage); + } + systemDiagnostics.setProvenanceRepositoryStorageUsage(provRepositoryUsage); + // get the garbage collection statistics final Map garbageCollection = new LinkedHashMap<>(garbageCollectors.size()); for (final GarbageCollectorMXBean garbageCollector : garbageCollectors) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index a445e49a40..ed42e9f327 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -2706,6 +2706,13 @@ public final class DtoFactory { contentRepositoryStorageUsageDtos.add(createStorageUsageDTO(entry.getKey(), entry.getValue())); } + // provenance disk usage + final Set provenanceRepositoryStorageUsageDtos = new LinkedHashSet<>(); + snapshot.setProvenanceRepositoryStorageUsage(provenanceRepositoryStorageUsageDtos); + for (final Map.Entry entry : sysDiagnostics.getProvenanceRepositoryStorageUsage().entrySet()) { + provenanceRepositoryStorageUsageDtos.add(createStorageUsageDTO(entry.getKey(), entry.getValue())); + } + // garbage collection final Set garbageCollectionDtos = new LinkedHashSet<>(); snapshot.setGarbageCollection(garbageCollectionDtos); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/cluster/cluster-content.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/cluster/cluster-content.jsp index 2b68060cec..75a43c5b8a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/cluster/cluster-content.jsp +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/cluster/cluster-content.jsp @@ -45,6 +45,9 @@
+
+
+
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/summary/system-diagnostics-dialog.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/summary/system-diagnostics-dialog.jsp index 15d5310746..4ad62a9b4e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/summary/system-diagnostics-dialog.jsp +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/summary/system-diagnostics-dialog.jsp @@ -74,7 +74,7 @@
Non-heap
- +
@@ -147,69 +147,73 @@
-
+
+
+
Available Processors:
+
+
+
+
+
+
+
+
+
+ Processor Load Average: +
+
+
+
+
+
+
+
-
Available Processors:
+
FlowFile Repository Storage
-
+
-
-
-
-
- Processor Load Average: -
-
+
Content Repository Storage
-
+
+
+
+
+
Provenance Repository Storage
+
+
-
-
-
FlowFile Repository Storage
-
-
+
+
+
NiFi
+
+
NiFi Version
+
Tag
+
Build Date/Time
+
Branch
+
Revision
+
+
+
+
Java
+
+
Version
+
Vendor
+
+
+
+
Operating System
+
+
Name
+
Version
+
Architecture
+
-
-
Content Repository Storage
-
-
-
-
-
- -
-
-
NiFi
-
-
NiFi Version
-
Tag
-
Build Date/Time
-
Branch
-
Revision
-
-
-
-
Java
-
-
Version
-
Vendor
-
-
-
-
Operating System
-
-
Name
-
Version
-
Architecture
-
-
-
-
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/summary.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/summary.css index 07bffc26f6..4e0e3f9909 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/summary.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/summary.css @@ -152,15 +152,15 @@ input.search-nodes { margin-top: 8px; } -#heap-table { +#heap-table, #non-heap-table { border-collapse: collapse; } -#heap-table tr{ +#heap-table tr, #non-heap-table tr { height: 20px; } -#heap-table span { +#heap-table span, #non-heap-table span { font-weight: bold; } @@ -191,6 +191,17 @@ input.search-nodes { margin-bottom: 10px; } +#provenance-repository-storage-usage-container { + height: 179px; + overflow-y: scroll; + border: 1px solid #ccc; + padding: 4px; +} + +#provenance-repository-storage-usage-container div.storage-usage { + margin-bottom: 10px; +} + .storage-usage-details { padding-top: 4px; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster-table.js index ac662496a2..0c94b31901 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/cluster/nf-cluster-table.js @@ -352,6 +352,89 @@ }] }; + var provenanceTab = { + name: 'Provenance Storage', + data: { + dataSet: 'systemDiagnostics', + update: updateProvenanceTableData + }, + tabContentId: 'cluster-provenance-tab-content', + tableId: 'cluster-provenance-table', + tableColumnModel: [ + { + id: 'node', + field: 'node', + name: 'Node Address', + sortable: true, + resizable: true, + formatter: nfCommon.genericValueFormatter + }, + { + id: 'provenanceRepoId', + field: 'provenanceRepoId', + name: 'Provenance Repository', + sortable: true, + resizable: true, + formatter: nfCommon.genericValueFormatter + }, + { + id: 'provenanceRepoTotal', + field: 'provenanceRepoTotal', + name: 'Total Space', + sortable: true, + resizable: true, + cssClass: 'cell-right', + headerCssClass: 'header-right', + formatter: nfCommon.genericValueFormatter + }, + { + id: 'provenanceRepoUsed', + field: 'provenanceRepoUsed', + name: 'Used Space', + sortable: true, + resizable: true, + cssClass: 'cell-right', + headerCssClass: 'header-right', + formatter: nfCommon.genericValueFormatter + }, + { + id: 'provenanceRepoFree', + field: 'provenanceRepoFree', + name: 'Free Space', + sortable: true, + resizable: true, + cssClass: 'cell-right', + headerCssClass: 'header-right', + formatter: nfCommon.genericValueFormatter + }, + { + id: 'provenanceRepoUtil', + field: 'provenanceRepoUtil', + name: 'Utilization', + sortable: true, + resizable: true, + cssClass: 'cell-right', + headerCssClass: 'header-right', + formatter: nfCommon.genericValueFormatter + } + ], + tableIdColumn: 'id', + tableOptions: commonTableOptions, + tableOnClick: null, + createTableOnEnter: null, + cleanUpTable: null, + init: commonTableInit, + onSort: sort, + onTabSelected: onSelectTab, + filterOptions: [{ + text: 'by address', + value: 'node' + }, { + text: 'by repository', + value: 'contentRepoId' + }] + }; + var versionTab = { name: 'Versions', data: { @@ -432,7 +515,7 @@ }] }; - var clusterTabs = [nodesTab, systemTab, jvmTab, flowFileTab, contentTab, versionTab]; + var clusterTabs = [nodesTab, systemTab, jvmTab, flowFileTab, contentTab, provenanceTab, versionTab]; var tabsByName = {}; var dataSetHandlers = {}; @@ -1200,6 +1283,39 @@ } } + /** + * Applies system diagnostics data to the Provenance Storage tab. + */ + function updateProvenanceTableData(systemDiagnosticsResponse) { + if (nfCommon.isDefinedAndNotNull(systemDiagnosticsResponse.systemDiagnostics) + && nfCommon.isDefinedAndNotNull(systemDiagnosticsResponse.systemDiagnostics.nodeSnapshots)) { + + var provenanceStorageTableRows = []; + systemDiagnosticsResponse.systemDiagnostics.nodeSnapshots.forEach(function (nodeSnapshot) { + var snapshot = nodeSnapshot.snapshot; + snapshot.provenanceRepositoryStorageUsage.forEach(function (provenanceRepoUsage) { + provenanceStorageTableRows.push({ + id: nodeSnapshot.nodeId + ':' + provenanceRepoUsage.identifier, + address: nodeSnapshot.address, + node: nodeSnapshot.address + ':' + nodeSnapshot.apiPort, + provenanceRepoId: provenanceRepoUsage.identifier, + provenanceRepoTotal: provenanceRepoUsage.totalSpace, + provenanceRepoUsed: provenanceRepoUsage.usedSpace, + provenanceRepoFree: provenanceRepoUsage.freeSpace, + provenanceRepoUtil: provenanceRepoUsage.utilization + }); + }); + }); + + provenanceTab.rowCount = provenanceStorageTableRows.length; + provenanceTab.dataView.setItems(provenanceStorageTableRows); + provenanceTab.dataView.reSort(); + provenanceTab.grid.invalidate(); + } else { + provenanceTab.rowCount = 0; + } + } + /** * Applies system diagnostics data to the Versions tab. */ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js index 92b622b6f9..4a1c954486 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js @@ -2415,6 +2415,19 @@ }); } + // provenance repo storage usage + var provenanceRepositoryUsageContainer = $('#provenance-repository-storage-usage-container').empty(); + if (nfCommon.isDefinedAndNotNull(aggregateSnapshot.provenanceRepositoryStorageUsage)) { + // sort the provenance repos + var sortedProvenanceRepositoryStorageUsage = aggregateSnapshot.provenanceRepositoryStorageUsage.sort(function (a, b) { + return a.identifier === b.identifier ? 0 : a.identifier > b.identifier ? 1 : -1; + }); + // add each to the UI + $.each(sortedProvenanceRepositoryStorageUsage, function (_, provenanceRepository) { + addStorageUsage(provenanceRepositoryUsageContainer, provenanceRepository); + }); + } + // Version var versionSpanSelectorToFieldMap = { '#version-nifi': aggregateSnapshot.versionInfo.niFiVersion, diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java index efe6140dee..604bb3ff8e 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java @@ -68,6 +68,7 @@ import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.RingBuffer; import org.apache.nifi.util.RingBuffer.ForEachEvaluator; +import org.apache.nifi.util.file.FileUtils; import org.apache.nifi.util.StopWatch; import org.apache.nifi.util.Tuple; import org.apache.nifi.util.timebuffer.CountSizeEntityAccess; @@ -521,6 +522,40 @@ public class PersistentProvenanceRepository implements ProvenanceRepository { return configuration; } + @Override + public Set getContainerNames() { + return new HashSet<>(configuration.getStorageDirectories().keySet()); + } + + @Override + public long getContainerCapacity(final String containerName) throws IOException { + Map map = configuration.getStorageDirectories(); + + File container = map.get(containerName); + if(container != null) { + long capacity = FileUtils.getContainerCapacity(container.toPath()); + if(capacity==0) { + throw new IOException("System returned total space of the partition for " + containerName + " is zero byte. " + + "Nifi can not create a zero sized provenance repository."); + } + return capacity; + } else { + throw new IllegalArgumentException("There is no defined container with name " + containerName); + } + } + + @Override + public long getContainerUsableSpace(String containerName) throws IOException { + Map map = configuration.getStorageDirectories(); + + File container = map.get(containerName); + if(container != null) { + return FileUtils.getContainerUsableSpace(container.toPath()); + } else { + throw new IllegalArgumentException("There is no defined container with name " + containerName); + } + } + private void recover() throws IOException { long maxId = -1L; long maxIndexedId = -1L; diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java index 4782dbe60a..e1299d0c30 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/WriteAheadProvenanceRepository.java @@ -17,10 +17,6 @@ package org.apache.nifi.provenance; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.RequestAction; import org.apache.nifi.authorization.resource.Authorizable; @@ -50,9 +46,18 @@ import org.apache.nifi.provenance.toc.TocWriter; import org.apache.nifi.provenance.util.CloseableUtil; import org.apache.nifi.reporting.Severity; import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.util.file.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** *

@@ -293,4 +298,38 @@ public class WriteAheadProvenanceRepository implements ProvenanceRepository { RepositoryConfiguration getConfig() { return this.config; } + + @Override + public Set getContainerNames() { + return new HashSet<>(config.getStorageDirectories().keySet()); + } + + @Override + public long getContainerCapacity(final String containerName) throws IOException { + Map map = config.getStorageDirectories(); + + File container = map.get(containerName); + if(container != null) { + long capacity = FileUtils.getContainerCapacity(container.toPath()); + if(capacity==0) { + throw new IOException("System returned total space of the partition for " + containerName + " is zero byte. " + + "Nifi can not create a zero sized provenance repository."); + } + return capacity; + } else { + throw new IllegalArgumentException("There is no defined container with name " + containerName); + } + } + + @Override + public long getContainerUsableSpace(String containerName) throws IOException { + Map map = config.getStorageDirectories(); + + File container = map.get(containerName); + if(container != null) { + return FileUtils.getContainerUsableSpace(container.toPath()); + } else { + throw new IllegalArgumentException("There is no defined container with name " + containerName); + } + } } diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java index c8511befde..dbac23ab9c 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/main/java/org/apache/nifi/provenance/VolatileProvenanceRepository.java @@ -50,6 +50,7 @@ import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; @@ -70,7 +71,10 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { // default property values public static final int DEFAULT_BUFFER_SIZE = 10000; + public static String CONTAINER_NAME = "in-memory"; + private final RingBuffer ringBuffer; + private final int maxSize; private final List searchableFields; private final List searchableAttributes; private final ExecutorService queryExecService; @@ -95,12 +99,13 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { scheduledExecService = null; authorizer = null; resourceFactory = null; + maxSize = DEFAULT_BUFFER_SIZE; } public VolatileProvenanceRepository(final NiFiProperties nifiProperties) { - final int bufferSize = nifiProperties.getIntegerProperty(BUFFER_SIZE, DEFAULT_BUFFER_SIZE); - ringBuffer = new RingBuffer<>(bufferSize); + maxSize = nifiProperties.getIntegerProperty(BUFFER_SIZE, DEFAULT_BUFFER_SIZE); + ringBuffer = new RingBuffer<>(maxSize); final String indexedFieldString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_FIELDS); final String indexedAttrString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_ATTRIBUTES); @@ -593,6 +598,21 @@ public class VolatileProvenanceRepository implements ProvenanceRepository { } } + @Override + public long getContainerCapacity(final String containerName) throws IOException { + return maxSize; + } + + @Override + public Set getContainerNames() { + return Collections.singleton(CONTAINER_NAME); + } + + @Override + public long getContainerUsableSpace(String containerName) throws IOException { + return maxSize - ringBuffer.getSize(); + } + private AsyncLineageSubmission submitLineageComputation(final Collection flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId) { final String userId = user.getIdentity(); final AsyncLineageSubmission result = new AsyncLineageSubmission(computationType, eventId, flowFileUuids, 1, userId);

Max: