[MRM-1580] system status page current scanner stats.

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1299905 13f79535-47bb-0310-9956-ffa450edef68

archiva-modules/archiva-base/archiva-repository-scanner/src/main/java/org/apache/archiva/repository/scanner/RepositoryScannerInstance.java

@@ -20,6 +20,10 @@ package org.apache.archiva.repository.scanner;
  */
 
 import org.apache.archiva.admin.model.beans.ManagedRepository;
+import org.apache.archiva.common.utils.BaseFile;
+import org.apache.archiva.consumers.InvalidRepositoryContentConsumer;
+import org.apache.archiva.consumers.KnownRepositoryContentConsumer;
+import org.apache.archiva.consumers.functors.ConsumerWantsFilePredicate;
 import org.apache.archiva.repository.scanner.functors.ConsumerProcessFileClosure;
 import org.apache.archiva.repository.scanner.functors.TriggerBeginScanClosure;
 import org.apache.archiva.repository.scanner.functors.TriggerScanCompletedClosure;
@@ -27,10 +31,6 @@ import org.apache.commons.collections.Closure;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.functors.IfClosure;
 import org.apache.commons.lang.SystemUtils;
-import org.apache.archiva.common.utils.BaseFile;
-import org.apache.archiva.consumers.InvalidRepositoryContentConsumer;
-import org.apache.archiva.consumers.KnownRepositoryContentConsumer;
-import org.apache.archiva.consumers.functors.ConsumerWantsFilePredicate;
 import org.codehaus.plexus.util.DirectoryWalkListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,8 +40,9 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
 /**
- * RepositoryScannerInstance 
+ * RepositoryScannerInstance
  *
  * @version $Id$
  */
@@ -49,7 +50,7 @@ public class RepositoryScannerInstance
     implements DirectoryWalkListener
 {
     private Logger log = LoggerFactory.getLogger( RepositoryScannerInstance.class );
-    
+
     /**
      * Consumers that process known content.
      */
@@ -82,8 +83,8 @@ public class RepositoryScannerInstance
         this.knownConsumers = knownConsumerList;
         this.invalidConsumers = invalidConsumerList;
 
-        consumerTimings = new HashMap<String,Long>();
+        consumerTimings = new HashMap<String, Long>();
-        consumerCounts = new HashMap<String,Long>();
+        consumerCounts = new HashMap<String, Long>();
 
         this.consumerProcessFile = new ConsumerProcessFileClosure();
         consumerProcessFile.setExecuteOnEntireRepo( true );
@@ -95,7 +96,8 @@ public class RepositoryScannerInstance
         stats = new RepositoryScanStatistics();
         stats.setRepositoryId( repository.getId() );
 
-        Closure triggerBeginScan = new TriggerBeginScanClosure( repository, new Date( System.currentTimeMillis() ), true );
+        Closure triggerBeginScan =
+            new TriggerBeginScanClosure( repository, new Date( System.currentTimeMillis() ), true );
 
         CollectionUtils.forAllDo( knownConsumerList, triggerBeginScan );
         CollectionUtils.forAllDo( invalidConsumerList, triggerBeginScan );
@@ -108,7 +110,8 @@ public class RepositoryScannerInstance
 
     public RepositoryScannerInstance( ManagedRepository repository,
                                       List<KnownRepositoryContentConsumer> knownContentConsumers,
-                                      List<InvalidRepositoryContentConsumer> invalidContentConsumers, long changesSince )
+                                      List<InvalidRepositoryContentConsumer> invalidContentConsumers,
+                                      long changesSince )
     {
         this( repository, knownContentConsumers, invalidContentConsumers );
 
@@ -134,7 +137,7 @@ public class RepositoryScannerInstance
 
     public void directoryWalkStarting( File basedir )
     {
-        log.info( "Walk Started: [" + this.repository.getId() + "] " + this.repository.getLocation() );
+        log.info( "Walk Started: [{}] {}", this.repository.getId(), this.repository.getLocation() );
         stats.triggerStart();
     }
 
@@ -175,7 +178,7 @@ public class RepositoryScannerInstance
         stats.setConsumerTimings( consumerTimings );
         stats.setConsumerCounts( consumerCounts );
 
-        log.info( "Walk Finished: [" + this.repository.getId() + "] " + this.repository.getLocation() );
+        log.info( "Walk Finished: [{}] {}", this.repository.getId(), this.repository.getLocation() );
         stats.triggerFinished();
     }
 
@@ -186,7 +189,7 @@ public class RepositoryScannerInstance
     {
         log.debug( "Repository Scanner: {}", message );
     }
-    
+
     public ManagedRepository getRepository()
     {
         return repository;

archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/ConsumerScanningStatistics.java

@@ -0,0 +1,71 @@
+package org.apache.archiva.rest.api.model;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+
+/**
+ * @author Olivier Lamy
+ */
+@XmlRootElement( name = "consumerScanningStatistics" )
+public class ConsumerScanningStatistics
+    implements Serializable
+{
+    private String consumerKey;
+
+    private long count;
+
+    private long time;
+
+    public ConsumerScanningStatistics()
+    {
+        // no op
+    }
+
+    public String getConsumerKey()
+    {
+        return consumerKey;
+    }
+
+    public void setConsumerKey( String consumerKey )
+    {
+        this.consumerKey = consumerKey;
+    }
+
+    public long getCount()
+    {
+        return count;
+    }
+
+    public void setCount( long count )
+    {
+        this.count = count;
+    }
+
+    public long getTime()
+    {
+        return time;
+    }
+
+    public void setTime( long time )
+    {
+        this.time = time;
+    }
+}

archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/model/RepositoryScannerStatistics.java

@@ -0,0 +1,119 @@
+package org.apache.archiva.rest.api.model;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.archiva.admin.model.beans.ManagedRepository;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * @author Olivier Lamy
+ */
+@XmlRootElement( name = "repositoryScannerStatistics" )
+public class RepositoryScannerStatistics
+    implements Serializable
+{
+    private ManagedRepository managedRepository;
+
+    //private RepositoryScanStatistics repositoryScanStatistics;
+
+    private List<ConsumerScanningStatistics> consumerScanningStatistics;
+
+    private long totalFileCount = 0;
+
+    private long newFileCount = 0;
+
+    public RepositoryScannerStatistics()
+    {
+        // no op
+    }
+
+    public ManagedRepository getManagedRepository()
+    {
+        return managedRepository;
+    }
+
+    public void setManagedRepository( ManagedRepository managedRepository )
+    {
+        this.managedRepository = managedRepository;
+    }
+
+    /*
+    public RepositoryScanStatistics getRepositoryScanStatistics()
+    {
+        return repositoryScanStatistics;
+    }
+
+    public void setRepositoryScanStatistics( RepositoryScanStatistics repositoryScanStatistics )
+    {
+        this.repositoryScanStatistics = repositoryScanStatistics;
+    }*/
+
+    /*public Map<String, Long> getConsumerCounts()
+    {
+        return consumerCounts;
+    }
+
+    public void setConsumerCounts( Map<String, Long> consumerCounts )
+    {
+        this.consumerCounts = consumerCounts;
+    }
+
+    public Map<String, Long> getConsumerTimings()
+    {
+        return consumerTimings;
+    }
+
+    public void setConsumerTimings( Map<String, Long> consumerTimings )
+    {
+        this.consumerTimings = consumerTimings;
+    } */
+
+    public List<ConsumerScanningStatistics> getConsumerScanningStatistics()
+    {
+        return consumerScanningStatistics;
+    }
+
+    public void setConsumerScanningStatistics( List<ConsumerScanningStatistics> consumerScanningStatistics )
+    {
+        this.consumerScanningStatistics = consumerScanningStatistics;
+    }
+
+    public long getTotalFileCount()
+    {
+        return totalFileCount;
+    }
+
+    public void setTotalFileCount( long totalFileCount )
+    {
+        this.totalFileCount = totalFileCount;
+    }
+
+    public long getNewFileCount()
+    {
+        return newFileCount;
+    }
+
+    public void setNewFileCount( long newFileCount )
+    {
+        this.newFileCount = newFileCount;
+    }
+}

archiva-modules/archiva-web/archiva-rest/archiva-rest-api/src/main/java/org/apache/archiva/rest/api/services/SystemStatusService.java

@@ -20,6 +20,7 @@ package org.apache.archiva.rest.api.services;
 
 import org.apache.archiva.rest.api.model.CacheEntry;
 import org.apache.archiva.rest.api.model.QueueEntry;
+import org.apache.archiva.rest.api.model.RepositoryScannerStatistics;
 import org.apache.archiva.security.common.ArchivaRoleConstants;
 import org.codehaus.plexus.redback.authorization.RedbackAuthorization;
 
@@ -80,4 +81,11 @@ public interface SystemStatusService
         throws ArchivaRestServiceException;
 
 
+    @Path( "repositoryScannerStatistics" )
+    @GET
+    @Produces( { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML } )
+    @RedbackAuthorization( permissions = ArchivaRoleConstants.OPERATION_MANAGE_CONFIGURATION )
+    List<RepositoryScannerStatistics> getRepositoryScannerStatistics()
+        throws ArchivaRestServiceException;
+
 }

archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/main/java/org/apache/archiva/rest/services/DefaultSystemStatusService.java

@@ -18,8 +18,12 @@ package org.apache.archiva.rest.services;
  * under the License.
  */
 
+import org.apache.archiva.repository.scanner.RepositoryScanner;
+import org.apache.archiva.repository.scanner.RepositoryScannerInstance;
 import org.apache.archiva.rest.api.model.CacheEntry;
+import org.apache.archiva.rest.api.model.ConsumerScanningStatistics;
 import org.apache.archiva.rest.api.model.QueueEntry;
+import org.apache.archiva.rest.api.model.RepositoryScannerStatistics;
 import org.apache.archiva.rest.api.services.ArchivaRestServiceException;
 import org.apache.archiva.rest.api.services.SystemStatusService;
 import org.codehaus.plexus.cache.Cache;
@@ -34,10 +38,12 @@ import javax.ws.rs.core.Response;
 import java.text.DecimalFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author Olivier Lamy
@@ -49,17 +55,18 @@ public class DefaultSystemStatusService
     implements SystemStatusService
 {
 
-    private ApplicationContext applicationContext;
 
     private Map<String, TaskQueue> queues = null;
 
-    Map<String, Cache> caches = null;
+    private Map<String, Cache> caches = null;
+
+    private RepositoryScanner scanner;
 
 
     @Inject
-    public DefaultSystemStatusService( ApplicationContext applicationContext )
+    public DefaultSystemStatusService( ApplicationContext applicationContext, RepositoryScanner scanner )
     {
-        this.applicationContext = applicationContext;
+        this.scanner = scanner;
 
         queues = getBeansOfType( applicationContext, TaskQueue.class );
 
@@ -150,4 +157,48 @@ public class DefaultSystemStatusService
         }
         return Boolean.TRUE;
     }
+
+    public List<RepositoryScannerStatistics> getRepositoryScannerStatistics()
+        throws ArchivaRestServiceException
+    {
+        Set<RepositoryScannerInstance> repositoryScannerInstances = scanner.getInProgressScans();
+        if ( repositoryScannerInstances.isEmpty() )
+        {
+            return Collections.emptyList();
+        }
+        List<RepositoryScannerStatistics> repositoryScannerStatisticsList =
+            new ArrayList<RepositoryScannerStatistics>( repositoryScannerInstances.size() );
+
+        for ( RepositoryScannerInstance instance : repositoryScannerInstances )
+        {
+            RepositoryScannerStatistics repositoryScannerStatistics = new RepositoryScannerStatistics();
+            repositoryScannerStatisticsList.add( repositoryScannerStatistics );
+            repositoryScannerStatistics.setManagedRepository( instance.getRepository() );
+            //repositoryScannerStatistics.setRepositoryScanStatistics( instance.getStatistics() );
+            //repositoryScannerStatistics.setConsumerCounts( new HashMap<String, Long>( instance.getConsumerCounts() ) );
+            //repositoryScannerStatistics.setConsumerTimings(
+            //    new HashMap<String, Long>( instance.getConsumerTimings() ) );
+            repositoryScannerStatistics.setNewFileCount( instance.getStats().getNewFileCount() );
+            repositoryScannerStatistics.setTotalFileCount( instance.getStats().getTotalFileCount() );
+            repositoryScannerStatistics.setConsumerScanningStatistics( mapConsumerScanningStatistics( instance ) );
+        }
+
+        return repositoryScannerStatisticsList;
+    }
+
+    private List<ConsumerScanningStatistics> mapConsumerScanningStatistics( RepositoryScannerInstance instance )
+    {
+        // FIXME take care of NPE here !!!
+        List<ConsumerScanningStatistics> ret =
+            new ArrayList<ConsumerScanningStatistics>( instance.getConsumerCounts().size() );
+        for ( Map.Entry<String, Long> entry : instance.getConsumerCounts().entrySet() )
+        {
+            ConsumerScanningStatistics consumerScanningStatistics = new ConsumerScanningStatistics();
+            consumerScanningStatistics.setConsumerKey( entry.getKey() );
+            consumerScanningStatistics.setCount( entry.getValue() );
+            consumerScanningStatistics.setTime( instance.getConsumerTimings().get( entry.getKey() ) );
+            ret.add( consumerScanningStatistics );
+        }
+        return ret;
+    }
 }

archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/general-admin.js

@@ -627,6 +627,34 @@ $(function() {
     });
   }
 
+  mapRepositoryScannerStatisticsList=function(data){
+    if(data!=null){
+      return $.isArray(data)? $.map(data,function(item){
+        return mapRepositoryScannerStatistics(item);
+      }):[data];
+    }
+    return [];
+  }
+
+
+  mapRepositoryScannerStatistics=function(data){
+    return new RepositoryScannerStatistics(mapManagedRepository(data.managedRepository),data.totalFileCount,
+                                            data.newFileCount,data.consumerScanningStatistics);
+  }
+
+  RepositoryScannerStatistics=function(managedRepository,totalFileCount,newFileCount,consumerScanningStatistics){
+    //private ManagedRepository managedRepository;
+    this.managedRepository=managedRepository
+
+    this.consumerScanningStatistics= consumerScanningStatistics;
+
+    //private long totalFileCount = 0;
+    this.totalFileCount=totalFileCount;
+
+    //private long newFileCount = 0;
+    this.newFileCount=newFileCount;
+  }
+
   displaySystemStatus=function(){
     screenChange();
     var mainContent=$("#main-content");
@@ -663,6 +691,18 @@ $(function() {
         }
     });
 
+    var dataStr='[{"managedRepository":{"id":"snapshots","name":"Archiva Managed Snapshot Repository","layout":"default","indexDirectory":null,"location":"/Users/olamy/dev/tests/archiva-appserver-base-test/data/repositories/snapshots","snapshots":true,"releases":false,"blockRedeployments":false,"cronExpression":"0 0,30 * * * ?","stagingRepository":null,"scanned":true,"daysOlder":30,"retentionCount":2,"deleteReleasedSnapshots":false,"stageRepoNeeded":false,"resetStats":false},"consumerCounts":{"create-missing-checksums":114,"duplicate-artifacts":12,"metadata-updater":114,"index-content":197,"create-archiva-metadata":113},"consumerTimings":{"create-missing-checksums":86,"duplicate-artifacts":929,"metadata-updater":263,"index-content":13,"create-archiva-metadata":11088},"totalFileCount":592,"newFileCount":592},{"managedRepository":{"id":"internal","name":"the Archiva Managed Internal Repository","layout":"default","indexDirectory":null,"location":"/Users/olamy/dev/tests/archiva-appserver-base-test/data/repositories/internal","snapshots":false,"releases":true,"blockRedeployments":true,"cronExpression":"0 */5 * * * ?","stagingRepository":null,"scanned":true,"daysOlder":30,"retentionCount":2,"deleteReleasedSnapshots":false,"stageRepoNeeded":false,"resetStats":false},"consumerCounts":{"create-missing-checksums":28,"duplicate-artifacts":28,"metadata-updater":28,"index-content":52,"create-archiva-metadata":28},"consumerTimings":{"create-missing-checksums":71,"duplicate-artifacts":4151,"metadata-updater":2645,"index-content":1,"create-archiva-metadata":3971},"totalFileCount":157,"newFileCount":157}]';
+    var data= mapRepositoryScannerStatisticsList( $.parseJSON(dataStr));
+    $.log("size:"+data.length);
+    mainContent.find("#status_scanning" ).html($("#status_scanning_tmpl").tmpl({repositoryScannerStatisticsList:data}));
+
+    $.ajax("restServices/archivaServices/systemStatusService/repositoryScannerStatistics", {
+        type: "GET",
+        success: function(data){
+          //mainContent.find("#status_scanning" ).html("#status_scanning_tmpl" ).tmpl(data);
+        }
+    });
+
     displayCacheEntries();
   }
 

archiva-modules/archiva-web/archiva-webapp-js/src/main/webapp/js/archiva/templates/general-admin.html

@@ -318,14 +318,14 @@
     <h4>${$.i18n.prop('system-status.header.queues')}</h4>
   </div>
   <div id="status_queues">
-      <img src="images/small-spinner.gif"/>
+    <img src="images/small-spinner.gif"/>
   </div>
 
   <div class="page-header">
     <h4>${$.i18n.prop('system-status.header.scanning')}</h4>
   </div>
   <div id="status_scanning">
-      <img src="images/small-spinner.gif"/>
+    <img src="images/small-spinner.gif"/>
   </div>
 
   <div class="page-header">
@@ -416,5 +416,30 @@
   </table>
 </script>
 
+<script id="status_scanning_tmpl" type="text/html">
+  {{if repositoryScannerStatisticsList.length == 0}}
+    <p>No scans in progress.</p>
+  {{else}}
+    <table>
+      <thead>
+        <tr>
+          <th>Repository</th>
+          <th>Files processed</th>
+          <th>New files</th>
+        </tr>
+      </thead>
+      <tbody>
+      {{each(i,repositoryScannerStatistics) repositoryScannerStatisticsList}}
+        <tr>
+          <td>${repositoryScannerStatistics.managedRepository.name()}</td>
+          <td>${repositoryScannerStatistics.totalFileCount}</td>
+          <td>${repositoryScannerStatistics.newFileCount}</td>
+        </tr>
+      {{/each}}
+      </tbody>
+    </table>
+  {{/if}}
+</script>
+