[MRM-77] ability to regenerate the report

git-svn-id: https://svn.apache.org/repos/asf/maven/archiva/trunk@441573 13f79535-47bb-0310-9956-ffa450edef68

archiva-core/src/main/java/org/apache/maven/archiva/scheduler/task/IndexerTask.java

@@ -31,22 +31,13 @@ import org.apache.maven.archiva.indexer.RepositoryArtifactIndexFactory;
 import org.apache.maven.archiva.indexer.RepositoryIndexException;
 import org.apache.maven.archiva.indexer.record.IndexRecordExistsArtifactFilter;
 import org.apache.maven.archiva.indexer.record.RepositoryIndexRecordFactory;
-import org.apache.maven.archiva.reporting.ArtifactReportProcessor;
+import org.apache.maven.archiva.reporting.ReportExecutor;
-import org.apache.maven.archiva.reporting.MetadataReportProcessor;
-import org.apache.maven.archiva.reporting.ReportingDatabase;
 import org.apache.maven.archiva.reporting.ReportingMetadataFilter;
-import org.apache.maven.archiva.reporting.ReportingStore;
 import org.apache.maven.archiva.reporting.ReportingStoreException;
 import org.apache.maven.archiva.scheduler.TaskExecutionException;
-import org.apache.maven.artifact.Artifact;
-import org.apache.maven.artifact.factory.ArtifactFactory;
 import org.apache.maven.artifact.repository.ArtifactRepository;
-import org.apache.maven.artifact.repository.metadata.RepositoryMetadata;
 import org.apache.maven.artifact.resolver.filter.AndArtifactFilter;
-import org.apache.maven.model.Model;
-import org.apache.maven.project.MavenProject;
 import org.apache.maven.project.MavenProjectBuilder;
-import org.apache.maven.project.ProjectBuildingException;
 import org.codehaus.plexus.logging.AbstractLogEnabled;
 
 import java.io.File;
@@ -89,21 +80,11 @@ public class IndexerTask
      */
     private Map artifactDiscoverers;
 
-    /**
-     * @plexus.requirement role="org.apache.maven.archiva.reporting.ArtifactReportProcessor"
-     */
-    private List artifactReports;
-
     /**
      * @plexus.requirement role="org.apache.maven.archiva.discoverer.MetadataDiscoverer"
      */
     private Map metadataDiscoverers;
 
-    /**
-     * @plexus.requirement role="org.apache.maven.archiva.reporting.MetadataReportProcessor"
-     */
-    private List metadataReports;
-
     /**
      * @plexus.requirement role-hint="standard"
      */
@@ -112,15 +93,10 @@ public class IndexerTask
     /**
      * @plexus.requirement
      */
-    private ArtifactFactory artifactFactory;
+    private ReportExecutor reportExecutor;
 
     private static final int ARTIFACT_BUFFER_SIZE = 1000;
 
-    /**
-     * @plexus.requirement
-     */
-    private ReportingStore reportingStore;
-
     public void execute()
         throws TaskExecutionException
     {
@@ -178,10 +154,6 @@ public class IndexerTask
 
                     ArtifactRepository repository = repoFactory.createRepository( repositoryConfiguration );
 
-                    getLogger().debug(
-                        "Reading previous report database from repository " + repositoryConfiguration.getName() );
-                    ReportingDatabase reporter = reportingStore.getReportsFromStore( repository );
-
                     // Discovery process
                     String layoutProperty = repositoryConfiguration.getLayout();
                     ArtifactDiscoverer discoverer = (ArtifactDiscoverer) artifactDiscoverers.get( layoutProperty );
@@ -210,12 +182,9 @@ public class IndexerTask
                             List currentArtifacts =
                                 artifacts.subList( j, end > artifacts.size() ? artifacts.size() : end );
 
-                            // run the reports
+                            // run the reports. Done intermittently to avoid losing track of what is indexed since
-                            runArtifactReports( currentArtifacts, reporter );
+                            // that is what the filter is based on.
-
+                            reportExecutor.runArtifactReports( currentArtifacts, repository );
-                            // store intermittently because if anything crashes out after indexing then we will have
-                            // lost track of these artifact's reports
-                            reportingStore.storeReports( reporter, repository );
 
                             index.indexArtifacts( currentArtifacts, recordFactory );
                         }
@@ -225,7 +194,8 @@ public class IndexerTask
                         flushProjectBuilderCacheHack();
                     }
 
-                    MetadataFilter metadataFilter = new ReportingMetadataFilter( reporter );
+                    MetadataFilter metadataFilter =
+                        new ReportingMetadataFilter( reportExecutor.getReportDatabase( repository ) );
 
                     MetadataDiscoverer metadataDiscoverer =
                         (MetadataDiscoverer) metadataDiscoverers.get( layoutProperty );
@@ -237,10 +207,8 @@ public class IndexerTask
                         getLogger().info( "Discovered " + metadata.size() + " unprocessed metadata files" );
 
                         // run the reports
-                        runMetadataReports( metadata, repository, reporter );
+                        reportExecutor.runMetadataReports( metadata, repository );
                     }
-
-                    reportingStore.storeReports( reporter, repository );
                 }
             }
         }
@@ -261,73 +229,6 @@ public class IndexerTask
         getLogger().info( "Finished repository indexing process in " + time + "ms" );
     }
 
-    private void runMetadataReports( List metadata, ArtifactRepository repository, ReportingDatabase reporter )
-    {
-        for ( Iterator i = metadata.iterator(); i.hasNext(); )
-        {
-            RepositoryMetadata repositoryMetadata = (RepositoryMetadata) i.next();
-
-            File file =
-                new File( repository.getBasedir(), repository.pathOfRemoteRepositoryMetadata( repositoryMetadata ) );
-            reporter.cleanMetadata( repositoryMetadata, file.lastModified() );
-
-            // TODO: should the report set be limitable by configuration?
-            runMetadataReports( repositoryMetadata, repository, reporter );
-        }
-    }
-
-    private void runMetadataReports( RepositoryMetadata repositoryMetadata, ArtifactRepository repository,
-                                     ReportingDatabase reporter )
-    {
-        for ( Iterator i = metadataReports.iterator(); i.hasNext(); )
-        {
-            MetadataReportProcessor report = (MetadataReportProcessor) i.next();
-
-            report.processMetadata( repositoryMetadata, repository, reporter );
-        }
-    }
-
-    private void runArtifactReports( List artifacts, ReportingDatabase reporter )
-    {
-        for ( Iterator i = artifacts.iterator(); i.hasNext(); )
-        {
-            Artifact artifact = (Artifact) i.next();
-
-            ArtifactRepository repository = artifact.getRepository();
-
-            Model model = null;
-            try
-            {
-                Artifact pomArtifact = artifactFactory.createProjectArtifact( artifact.getGroupId(),
-                                                                              artifact.getArtifactId(),
-                                                                              artifact.getVersion() );
-                MavenProject project =
-                    projectBuilder.buildFromRepository( pomArtifact, Collections.EMPTY_LIST, repository );
-
-                model = project.getModel();
-            }
-            catch ( ProjectBuildingException e )
-            {
-                reporter.addWarning( artifact, "Error reading project model: " + e );
-            }
-
-            reporter.removeArtifact( artifact );
-
-            runArtifactReports( artifact, model, reporter );
-        }
-    }
-
-    private void runArtifactReports( Artifact artifact, Model model, ReportingDatabase reporter )
-    {
-        // TODO: should the report set be limitable by configuration?
-        for ( Iterator i = artifactReports.iterator(); i.hasNext(); )
-        {
-            ArtifactReportProcessor report = (ArtifactReportProcessor) i.next();
-
-            report.processArtifact( artifact, model, reporter );
-        }
-    }
-
     public void executeNowIfNeeded()
         throws TaskExecutionException
     {

archiva-reports-standard/src/main/java/org/apache/maven/archiva/reporting/DefaultReportExecutor.java

@@ -0,0 +1,223 @@
+package org.apache.maven.archiva.reporting;
+
+/*
+ * Copyright 2005-2006 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.maven.archiva.discoverer.ArtifactDiscoverer;
+import org.apache.maven.archiva.discoverer.DiscovererException;
+import org.apache.maven.archiva.discoverer.MetadataDiscoverer;
+import org.apache.maven.archiva.discoverer.filter.AcceptAllMetadataFilter;
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.factory.ArtifactFactory;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.repository.layout.ArtifactRepositoryLayout;
+import org.apache.maven.artifact.repository.layout.DefaultRepositoryLayout;
+import org.apache.maven.artifact.repository.layout.LegacyRepositoryLayout;
+import org.apache.maven.artifact.repository.metadata.RepositoryMetadata;
+import org.apache.maven.artifact.resolver.filter.ArtifactFilter;
+import org.apache.maven.model.Model;
+import org.apache.maven.project.MavenProject;
+import org.apache.maven.project.MavenProjectBuilder;
+import org.apache.maven.project.ProjectBuildingException;
+import org.codehaus.plexus.logging.AbstractLogEnabled;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Report executor implementation.
+ *
+ * @plexus.component
+ */
+public class DefaultReportExecutor
+    extends AbstractLogEnabled
+    implements ReportExecutor
+{
+    /**
+     * @plexus.requirement
+     */
+    private MavenProjectBuilder projectBuilder;
+
+    /**
+     * @plexus.requirement
+     */
+    private ReportingStore reportingStore;
+
+    /**
+     * @plexus.requirement
+     */
+    private ArtifactFactory artifactFactory;
+
+    /**
+     * @todo replace with a ReportGroup that is identified as "health" and has requirements on the specific health reports
+     * @plexus.requirement role="org.apache.maven.archiva.reporting.ArtifactReportProcessor"
+     */
+    private List artifactReports;
+
+    /**
+     * @plexus.requirement role="org.apache.maven.archiva.reporting.MetadataReportProcessor"
+     */
+    private List metadataReports;
+
+    /**
+     * @plexus.requirement role="org.apache.maven.archiva.discoverer.ArtifactDiscoverer"
+     */
+    private Map artifactDiscoverers;
+
+    /**
+     * @plexus.requirement role="org.apache.maven.archiva.discoverer.MetadataDiscoverer"
+     */
+    private Map metadataDiscoverers;
+
+    public void runMetadataReports( List metadata, ArtifactRepository repository )
+        throws ReportingStoreException
+    {
+        ReportingDatabase reporter = getReportDatabase( repository );
+
+        for ( Iterator i = metadata.iterator(); i.hasNext(); )
+        {
+            RepositoryMetadata repositoryMetadata = (RepositoryMetadata) i.next();
+
+            File file =
+                new File( repository.getBasedir(), repository.pathOfRemoteRepositoryMetadata( repositoryMetadata ) );
+            reporter.cleanMetadata( repositoryMetadata, file.lastModified() );
+
+            // TODO: should the report set be limitable by configuration?
+            runMetadataReports( repositoryMetadata, repository, reporter );
+        }
+
+        reportingStore.storeReports( reporter, repository );
+    }
+
+    private void runMetadataReports( RepositoryMetadata repositoryMetadata, ArtifactRepository repository,
+                                     ReportingDatabase reporter )
+    {
+        for ( Iterator i = metadataReports.iterator(); i.hasNext(); )
+        {
+            MetadataReportProcessor report = (MetadataReportProcessor) i.next();
+
+            report.processMetadata( repositoryMetadata, repository, reporter );
+        }
+    }
+
+    public void runArtifactReports( List artifacts, ArtifactRepository repository )
+        throws ReportingStoreException
+    {
+        ReportingDatabase reporter = getReportDatabase( repository );
+
+        for ( Iterator i = artifacts.iterator(); i.hasNext(); )
+        {
+            Artifact artifact = (Artifact) i.next();
+
+            Model model = null;
+            try
+            {
+                Artifact pomArtifact = artifactFactory.createProjectArtifact( artifact.getGroupId(),
+                                                                              artifact.getArtifactId(),
+                                                                              artifact.getVersion() );
+                MavenProject project =
+                    projectBuilder.buildFromRepository( pomArtifact, Collections.EMPTY_LIST, repository );
+
+                model = project.getModel();
+            }
+            catch ( ProjectBuildingException e )
+            {
+                reporter.addWarning( artifact, "Error reading project model: " + e );
+            }
+
+            reporter.removeArtifact( artifact );
+
+            runArtifactReports( artifact, model, reporter );
+        }
+
+        reportingStore.storeReports( reporter, repository );
+    }
+
+    public ReportingDatabase getReportDatabase( ArtifactRepository repository )
+        throws ReportingStoreException
+    {
+        getLogger().debug( "Reading previous report database from repository " + repository.getId() );
+        return reportingStore.getReportsFromStore( repository );
+    }
+
+    public void runReports( ArtifactRepository repository, List blacklistedPatterns, ArtifactFilter filter )
+        throws DiscovererException, ReportingStoreException
+    {
+        // Flush (as in toilet, not store) the report database
+        reportingStore.removeReportDatabase( repository );
+
+        // Discovery process
+        String layoutProperty = getRepositoryLayout( repository.getLayout() );
+        ArtifactDiscoverer discoverer = (ArtifactDiscoverer) artifactDiscoverers.get( layoutProperty );
+
+        // Save some memory by not tracking paths we won't use
+        // TODO: Plexus CDC should be able to inject this configuration
+        discoverer.setTrackOmittedPaths( false );
+
+        List artifacts = discoverer.discoverArtifacts( repository, blacklistedPatterns, filter );
+
+        if ( !artifacts.isEmpty() )
+        {
+            getLogger().info( "Discovered " + artifacts.size() + " artifacts" );
+
+            // run the reports
+            runArtifactReports( artifacts, repository );
+        }
+
+        MetadataDiscoverer metadataDiscoverer = (MetadataDiscoverer) metadataDiscoverers.get( layoutProperty );
+        List metadata =
+            metadataDiscoverer.discoverMetadata( repository, blacklistedPatterns, new AcceptAllMetadataFilter() );
+
+        if ( !metadata.isEmpty() )
+        {
+            getLogger().info( "Discovered " + metadata.size() + " metadata files" );
+
+            // run the reports
+            runMetadataReports( metadata, repository );
+        }
+    }
+
+    private String getRepositoryLayout( ArtifactRepositoryLayout layout )
+    {
+        // gross limitation that there is no reverse lookup of the hint for the layout.
+        if ( layout.getClass().equals( DefaultRepositoryLayout.class ) )
+        {
+            return "default";
+        }
+        else if ( layout.getClass().equals( LegacyRepositoryLayout.class ) )
+        {
+            return "legacy";
+        }
+        else
+        {
+            throw new IllegalArgumentException( "Unknown layout: " + layout );
+        }
+    }
+
+    private void runArtifactReports( Artifact artifact, Model model, ReportingDatabase reporter )
+    {
+        // TODO: should the report set be limitable by configuration?
+        for ( Iterator i = artifactReports.iterator(); i.hasNext(); )
+        {
+            ArtifactReportProcessor report = (ArtifactReportProcessor) i.next();
+
+            report.processArtifact( artifact, model, reporter );
+        }
+    }
+}

archiva-reports-standard/src/main/java/org/apache/maven/archiva/reporting/DefaultReportingStore.java

@@ -120,4 +120,9 @@ public class DefaultReportingStore
             IOUtil.close( fileWriter );
         }
     }
+
+    public void removeReportDatabase( ArtifactRepository repository )
+    {
+        reports.remove( repository );
+    }
 }

archiva-reports-standard/src/main/java/org/apache/maven/archiva/reporting/ReportExecutor.java

@@ -0,0 +1,74 @@
+package org.apache.maven.archiva.reporting;
+
+/*
+ * Copyright 2005-2006 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.maven.archiva.discoverer.DiscovererException;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.resolver.filter.ArtifactFilter;
+
+import java.util.List;
+
+/**
+ * Executes a report or report group.
+ */
+public interface ReportExecutor
+{
+    /**
+     * Plexus component role name.
+     */
+    String ROLE = ReportExecutor.class.getName();
+
+    /**
+     * Run reports on a set of metadata.
+     *
+     * @param metadata   the RepositoryMetadata objects to report on
+     * @param repository the repository that they come from
+     * @throws ReportingStoreException if there is a problem reading/writing the report database
+     */
+    public void runMetadataReports( List metadata, ArtifactRepository repository )
+        throws ReportingStoreException;
+
+    /**
+     * Run reports on a set of artifacts.
+     *
+     * @param artifacts  the Artifact objects to report on
+     * @param repository the repository that they come from
+     * @throws ReportingStoreException if there is a problem reading/writing the report database
+     */
+    public void runArtifactReports( List artifacts, ArtifactRepository repository )
+        throws ReportingStoreException;
+
+    /**
+     * Get the report database in use for a given repository.
+     *
+     * @param repository the repository
+     * @return the report database
+     * @throws ReportingStoreException if there is a problem reading the report database
+     */
+    ReportingDatabase getReportDatabase( ArtifactRepository repository )
+        throws ReportingStoreException;
+
+    /**
+     * Run the artifact and metadata reports for the repository. The artifacts and metadata will be discovered.
+     *
+     * @param repository          the repository to run from
+     * @param blacklistedPatterns the patterns to exclude during discovery
+     * @param filter              the filter to use during discovery to get a consistent list of artifacts
+     */
+    public void runReports( ArtifactRepository repository, List blacklistedPatterns, ArtifactFilter filter )
+        throws DiscovererException, ReportingStoreException;
+}

archiva-reports-standard/src/main/java/org/apache/maven/archiva/reporting/ReportingDatabase.java

@@ -278,4 +278,5 @@ public class ReportingDatabase
     {
         return repository;
     }
+
 }

archiva-reports-standard/src/main/java/org/apache/maven/archiva/reporting/ReportingStore.java

@@ -50,4 +50,11 @@ public interface ReportingStore
      */
     void storeReports( ReportingDatabase database, ArtifactRepository repository )
         throws ReportingStoreException;
+
+    /**
+     * Remove the report database from the memory cache.
+     *
+     * @param repository the repository of the database to remove
+     */
+    void removeReportDatabase( ArtifactRepository repository );
 }

archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/ReportsAction.java

@@ -21,9 +21,13 @@ import org.apache.maven.archiva.configuration.Configuration;
 import org.apache.maven.archiva.configuration.ConfigurationStore;
 import org.apache.maven.archiva.configuration.ConfiguredRepositoryFactory;
 import org.apache.maven.archiva.configuration.RepositoryConfiguration;
+import org.apache.maven.archiva.discoverer.filter.AcceptAllArtifactFilter;
+import org.apache.maven.archiva.discoverer.filter.SnapshotArtifactFilter;
+import org.apache.maven.archiva.reporting.ReportExecutor;
 import org.apache.maven.archiva.reporting.ReportingDatabase;
 import org.apache.maven.archiva.reporting.ReportingStore;
 import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.resolver.filter.ArtifactFilter;
 
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -54,6 +58,13 @@ public class ReportsAction
 
     private List databases;
 
+    private String repositoryId;
+
+    /**
+     * @plexus.requirement
+     */
+    private ReportExecutor executor;
+
     public String execute()
         throws Exception
     {
@@ -74,6 +85,51 @@ public class ReportsAction
         return SUCCESS;
     }
 
+    public String runReport()
+        throws Exception
+    {
+        // TODO: this should be one that runs in the background - see the showcase
+
+        Configuration configuration = configurationStore.getConfigurationFromStore();
+
+        RepositoryConfiguration repositoryConfiguration = configuration.getRepositoryById( repositoryId );
+        ArtifactRepository repository = factory.createRepository( repositoryConfiguration );
+
+        List blacklistedPatterns = new ArrayList();
+        if ( repositoryConfiguration.getBlackListPatterns() != null )
+        {
+            blacklistedPatterns.addAll( repositoryConfiguration.getBlackListPatterns() );
+        }
+        if ( configuration.getGlobalBlackListPatterns() != null )
+        {
+            blacklistedPatterns.addAll( configuration.getGlobalBlackListPatterns() );
+        }
+
+        ArtifactFilter filter;
+        if ( repositoryConfiguration.isIncludeSnapshots() )
+        {
+            filter = new AcceptAllArtifactFilter();
+        }
+        else
+        {
+            filter = new SnapshotArtifactFilter();
+        }
+
+        executor.runReports( repository, blacklistedPatterns, filter );
+
+        return execute();
+    }
+
+    public String getRepositoryId()
+    {
+        return repositoryId;
+    }
+
+    public void setRepositoryId( String repositoryId )
+    {
+        this.repositoryId = repositoryId;
+    }
+
     public List getDatabases()
     {
         return databases;

archiva-webapp/src/main/webapp/WEB-INF/jsp/reports/reports.jsp

@@ -32,6 +32,19 @@
 
 <ww:set name="databases" value="databases"/>
 <c:forEach items="${databases}" var="database">
+<div>
+<div style="float: right">
+    <%-- TODO!
+  <a href="#">Repair all</a>
+  |
+    --%>
+  <c:set var="url">
+    <ww:url action="reports" namespace="/" method="runReport">
+      <ww:param name="repositoryId" value="%{'${database.repository.id}'}"/>
+    </ww:url>
+  </c:set>
+  <a href="${url}">Regenerate Report</a>
+</div>
 <h2>Repository: ${database.repository.name}</h2>
 
 <p>
@@ -40,9 +53,6 @@
     ${database.numFailures}
   <img src="<c:url value="/images/icon_warning_sml.gif"/>" width="15" height="15" alt=""/>
     ${database.numWarnings}
-    <%-- TODO!
-          (<a href="#">Repair all</a>)
-    --%>
 </p>
 
   <%-- TODO! factor out common parts, especially artifact rendering tag --%>
@@ -174,6 +184,7 @@
     </p>
   </c:if>
 </c:if>
+</div>
 </c:forEach>
 </div>
 

design/white-site/src/site/xdoc/reports.xml

@@ -10,6 +10,12 @@
     </div>
 
     <div id="contentArea">
+      <div>
+        <div style="float: right">
+          <a href="#">Repair all</a>
+          |
+          <a href="#">Regenerate Report</a>
+        </div>
         <h2>Repository 1</h2>
 
         <p>
@@ -18,9 +24,6 @@
           2
           <img src="images/icon_warning_sml.gif" width="15" height="15" alt=""/>
           1
-        (
-        <a href="#">Repair all</a>
-        )
         </p>
 
         <h3>Artifacts</h3>
@@ -71,7 +74,15 @@
         <p>
           <b>... more ...</b>
         </p>
+      </div>
 
+
+      <div>
+        <div style="float: right">
+          <a href="#">Repair all</a>
+          |
+          <a href="#">Regenerate Report</a>
+        </div>
         <h2>Repository 2</h2>
 
         <p>
@@ -80,9 +91,6 @@
           2
           <img src="images/icon_warning_sml.gif" width="15" height="15" alt=""/>
           0
-        (
-        <a href="#">Repair all</a>
-        )
         </p>
 
         <h3>Artifacts</h3>
@@ -128,6 +136,8 @@
           </span>
         </p>
       </div>
+
+    </div>
   </body>
 </document>