[MRM-555] scanner checks timestamps, but some consumers will want to consume artifacts regardless

git-svn-id: https://svn.apache.org/repos/asf/maven/archiva/trunk@586827 13f79535-47bb-0310-9956-ffa450edef68

archiva-base/archiva-consumers/archiva-consumer-api/src/main/java/org/apache/maven/archiva/consumers/AbstractMonitoredConsumer.java

@@ -94,4 +94,9 @@ public abstract class AbstractMonitoredConsumer
             }
         }
     }
+
+    public boolean isProcessUnmodified()
+    {
+        return false;
+    }
 }

archiva-base/archiva-consumers/archiva-consumer-api/src/main/java/org/apache/maven/archiva/consumers/RepositoryContentConsumer.java

@@ -89,4 +89,11 @@ public interface RepositoryContentConsumer extends BaseConsumer
      * @todo! this is never called by the RepositoryScannerInstance
      */
     public void completeScan();
+
+    /**
+     * Whether the consumer should process files that have not been modified since the time passed in to the scan
+     * method.
+     * @return whether to process the unmodified files
+     */
+    boolean isProcessUnmodified();
 }

archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/repository/RepositoryPurgeConsumer.java

@@ -216,4 +216,9 @@ public class RepositoryPurgeConsumer
         initIncludes();
     }
 
+    public boolean isProcessUnmodified()
+    {
+        // we need to check all files for deletion, especially if not modified
+        return true;
+    }
 }

archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/DefaultRepositoryScanner.java

@@ -112,8 +112,7 @@ public class DefaultRepositoryScanner
 
         // Setup the Scan Instance
         RepositoryScannerInstance scannerInstance = new RepositoryScannerInstance( repository, knownContentConsumers,
-                                                                                   invalidContentConsumers, getLogger() );
+                                                                                   invalidContentConsumers, getLogger(), changesSince );
-        scannerInstance.setOnlyModifiedAfterTimestamp( changesSince );
 
         dirWalker.addDirectoryWalkListener( scannerInstance );
 

archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerInstance.java

@@ -59,7 +59,7 @@ public class RepositoryScannerInstance
 
     private RepositoryScanStatistics stats;
 
-    private long onlyModifiedAfterTimestamp = 0;
+    private long changesSince = 0;
 
     private ConsumerProcessFileClosure consumerProcessFile;
 
@@ -93,6 +93,18 @@ public class RepositoryScannerInstance
         }
     }
 
+    public RepositoryScannerInstance( ManagedRepositoryConfiguration repository,
+                                      List<KnownRepositoryContentConsumer> knownContentConsumers,
+                                      List<InvalidRepositoryContentConsumer> invalidContentConsumers, Logger logger,
+                                      long changesSince )
+    {
+        this( repository, knownContentConsumers, invalidContentConsumers, logger );
+
+        consumerWantsFile.setChangesSince( changesSince );
+
+        this.changesSince = changesSince;
+    }
+
     public RepositoryScanStatistics getStatistics()
     {
         return stats;
@@ -111,15 +123,12 @@ public class RepositoryScannerInstance
         stats.increaseFileCount();
 
         // Timestamp finished points to the last successful scan, not this current one.
-        if ( file.lastModified() < onlyModifiedAfterTimestamp )
+        if ( file.lastModified() >= changesSince )
         {
-            // Skip file as no change has occured.
+            stats.increaseNewFileCount();
-            logger.debug( "Skipping, No Change: " + file.getAbsolutePath() );
-            return;
         }
 
-        stats.increaseNewFileCount();
+        // consume files regardless - the predicate will check the timestamp
-
         BaseFile basefile = new BaseFile( repository.getLocation(), file );
         
         consumerProcessFile.setBasefile( basefile );
@@ -141,16 +150,6 @@ public class RepositoryScannerInstance
         stats.triggerFinished();
     }
 
-    public long getOnlyModifiedAfterTimestamp()
-    {
-        return onlyModifiedAfterTimestamp;
-    }
-
-    public void setOnlyModifiedAfterTimestamp( long onlyModifiedAfterTimestamp )
-    {
-        this.onlyModifiedAfterTimestamp = onlyModifiedAfterTimestamp;
-    }
-
     /**
      * Debug method from DirectoryWalker.
      */

archiva-base/archiva-repository-layer/src/main/java/org/apache/maven/archiva/repository/scanner/functors/ConsumerWantsFilePredicate.java

@@ -42,6 +42,8 @@ public class ConsumerWantsFilePredicate
 
     private int wantedFileCount = 0;
 
+    private long changesSince = 0;
+
     public boolean evaluate( Object object )
     {
         boolean satisfies = false;
@@ -52,7 +54,19 @@ public class ConsumerWantsFilePredicate
             if ( wantsFile( consumer, StringUtils.replace( basefile.getRelativePath(), "\\", "/" ) ) )
             {
                 satisfies = true;
+                
+                // regardless of the timestamp, we record that it was wanted so it doesn't get counted as invalid
                 wantedFileCount++;
+
+                if ( !consumer.isProcessUnmodified() )
+                {
+                    // Timestamp finished points to the last successful scan, not this current one.
+                    if ( basefile.lastModified() < changesSince )
+                    {
+                        // Skip file as no change has occured.
+                        satisfies = false;
+                    }
+                }
             }
         }
 
@@ -114,4 +128,9 @@ public class ConsumerWantsFilePredicate
         // Not included, and Not excluded?  Default to EXCLUDE.
         return false;
     }
+
+    public void setChangesSince( long changesSince )
+    {
+        this.changesSince = changesSince;
+    }
 }

archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/KnownScanConsumer.java

@@ -42,6 +42,8 @@ public class KnownScanConsumer
 
     private List includes = new ArrayList();
 
+    private boolean processUnmodified = false;
+
     public List getExcludes()
     {
         return null;
@@ -99,4 +101,14 @@ public class KnownScanConsumer
     {
         return false;
     }
+
+    public boolean isProcessUnmodified()
+    {
+        return processUnmodified;
+    }
+
+    public void setProcessUnmodified( boolean processUnmodified )
+    {
+        this.processUnmodified = processUnmodified;
+    }
 }

archiva-base/archiva-repository-layer/src/test/java/org/apache/maven/archiva/repository/scanner/RepositoryScannerTest.java

@@ -19,17 +19,23 @@ package org.apache.maven.archiva.repository.scanner;
  * under the License.
  */
 
+import org.apache.commons.io.FileUtils;
 import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
 import org.apache.maven.archiva.model.RepositoryContentStatistics;
 import org.apache.maven.archiva.repository.AbstractRepositoryLayerTestCase;
 
 import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
 
 /**
- * RepositoryScannerTest 
+ * RepositoryScannerTest
  *
  * @author <a href="mailto:joakime@apache.org">Joakim Erdfelt</a>
  * @version $Id$
@@ -37,23 +43,44 @@ import java.util.List;
 public class RepositoryScannerTest
     extends AbstractRepositoryLayerTestCase
 {
-    private static final String[] ARTIFACT_PATTERNS = new String[] {
+    private static final String[] ARTIFACT_PATTERNS =
-        "**/*.jar",
+        new String[]{"**/*.jar", "**/*.pom", "**/*.rar", "**/*.zip", "**/*.war", "**/*.tar.gz"};
-        "**/*.pom",
-        "**/*.rar",
-        "**/*.zip",
-        "**/*.war",
-        "**/*.tar.gz" };
 
     private ManagedRepositoryConfiguration createDefaultRepository()
     {
         File repoDir = new File( getBasedir(), "src/test/repositories/default-repository" );
 
         assertTrue( "Default Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() );
-        
-        ManagedRepositoryConfiguration repo = createRepository( "testDefaultRepo", "Test Default Repository", repoDir );
 
-        return repo;
+        return createRepository( "testDefaultRepo", "Test Default Repository", repoDir );
+    }
+
+    private ManagedRepositoryConfiguration createSimpleRepository()
+        throws IOException, ParseException
+    {
+        File srcDir = new File( getBasedir(), "src/test/repositories/simple-repository" );
+
+        File repoDir = getTestFile( "target/test-repos/simple-repository" );
+
+        FileUtils.deleteDirectory( repoDir );
+
+        FileUtils.copyDirectory( srcDir, repoDir );
+
+        File repoFile = new File( repoDir,
+                                  "groupId/snapshot-artifact/1.0-alpha-1-SNAPSHOT/snapshot-artifact-1.0-alpha-1-20050611.202024-1.pom" );
+        repoFile.setLastModified( getTimestampAsMillis( "20050611.202024" ) );
+
+        assertTrue( "Simple Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() );
+
+        return createRepository( "testSimpleRepo", "Test Simple Repository", repoDir );
+    }
+
+    private static long getTimestampAsMillis( String timestamp )
+        throws ParseException
+    {
+        SimpleDateFormat fmt = new SimpleDateFormat( "yyyyMMdd.HHmmss", Locale.US );
+        fmt.setTimeZone( TimeZone.getTimeZone( "UTC" ) );
+        return fmt.parse( timestamp ).getTime();
     }
 
     private ManagedRepositoryConfiguration createLegacyRepository()
@@ -72,8 +99,8 @@ public class RepositoryScannerTest
     {
         if ( actualCount < minimumHitCount )
         {
-            fail( "Minimum hit count on " + msg + " not satisfied.  Expected more than <" + minimumHitCount
+            fail( "Minimum hit count on " + msg + " not satisfied.  Expected more than <" + minimumHitCount +
-                + ">, but actually got <" + actualCount + ">." );
+                ">, but actually got <" + actualCount + ">." );
         }
     }
 
@@ -90,23 +117,14 @@ public class RepositoryScannerTest
         return ignores;
     }
 
-    public void testDefaultRepositoryScanner()
+    public void testTimestampRepositoryScanner()
         throws Exception
     {
-        ManagedRepositoryConfiguration repository = createDefaultRepository();
+        ManagedRepositoryConfiguration repository = createSimpleRepository();
 
         List knownConsumers = new ArrayList();
         KnownScanConsumer consumer = new KnownScanConsumer();
-        consumer.setIncludes( new String[] {
+        consumer.setIncludes( ARTIFACT_PATTERNS );
-            "**/*.jar",
-            "**/*.war",
-            "**/*.pom",
-            "**/maven-metadata.xml",
-            "**/*-site.xml",
-            "**/*.zip",
-            "**/*.tar.gz",
-            "**/*.sha1",
-            "**/*.md5" } );
         knownConsumers.add( consumer );
 
         List invalidConsumers = new ArrayList();
@@ -114,8 +132,86 @@ public class RepositoryScannerTest
         invalidConsumers.add( badconsumer );
 
         RepositoryScanner scanner = lookupRepositoryScanner();
-        RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers,
+
-                                                          getIgnoreList(), RepositoryScanner.FRESH_SCAN );
+        RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(),
+                                                          getTimestampAsMillis( "20061101.000000" ) );
+
+        assertNotNull( "Stats should not be null.", stats );
+        assertEquals( "Stats.totalFileCount", 4, stats.getTotalFileCount() );
+        assertEquals( "Stats.newFileCount", 3, stats.getNewFileCount() );
+        assertEquals( "Processed Count", 2, consumer.getProcessCount() );
+        assertEquals( "Processed Count (of invalid items)", 1, badconsumer.getProcessCount() );
+    }
+
+    public void testTimestampRepositoryScannerFreshScan()
+        throws Exception
+    {
+        ManagedRepositoryConfiguration repository = createSimpleRepository();
+
+        List knownConsumers = new ArrayList();
+        KnownScanConsumer consumer = new KnownScanConsumer();
+        consumer.setIncludes( ARTIFACT_PATTERNS );
+        knownConsumers.add( consumer );
+
+        List invalidConsumers = new ArrayList();
+        InvalidScanConsumer badconsumer = new InvalidScanConsumer();
+        invalidConsumers.add( badconsumer );
+
+        RepositoryScanner scanner = lookupRepositoryScanner();
+        RepositoryContentStatistics stats =
+            scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
+
+        assertNotNull( "Stats should not be null.", stats );
+        assertEquals( "Stats.totalFileCount", 4, stats.getTotalFileCount() );
+        assertEquals( "Stats.newFileCount", 4, stats.getNewFileCount() );
+        assertEquals( "Processed Count", 3, consumer.getProcessCount() );
+        assertEquals( "Processed Count (of invalid items)", 1, badconsumer.getProcessCount() );
+    }
+
+    public void testTimestampRepositoryScannerProcessUnmodified()
+        throws Exception
+    {
+        ManagedRepositoryConfiguration repository = createSimpleRepository();
+
+        List knownConsumers = new ArrayList();
+        KnownScanConsumer consumer = new KnownScanConsumer();
+        consumer.setProcessUnmodified( true );
+        consumer.setIncludes( ARTIFACT_PATTERNS );
+        knownConsumers.add( consumer );
+
+        List invalidConsumers = new ArrayList();
+        InvalidScanConsumer badconsumer = new InvalidScanConsumer();
+        invalidConsumers.add( badconsumer );
+
+        RepositoryScanner scanner = lookupRepositoryScanner();
+        RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(),
+                                                          getTimestampAsMillis( "20061101.000000" ) );
+
+        assertNotNull( "Stats should not be null.", stats );
+        assertEquals( "Stats.totalFileCount", 4, stats.getTotalFileCount() );
+        assertEquals( "Stats.newFileCount", 3, stats.getNewFileCount() );
+        assertEquals( "Processed Count", 3, consumer.getProcessCount() );
+        assertEquals( "Processed Count (of invalid items)", 1, badconsumer.getProcessCount() );
+    }
+
+    public void testDefaultRepositoryScanner()
+        throws Exception
+    {
+        ManagedRepositoryConfiguration repository = createDefaultRepository();
+
+        List knownConsumers = new ArrayList();
+        KnownScanConsumer consumer = new KnownScanConsumer();
+        consumer.setIncludes( new String[]{"**/*.jar", "**/*.war", "**/*.pom", "**/maven-metadata.xml", "**/*-site.xml",
+            "**/*.zip", "**/*.tar.gz", "**/*.sha1", "**/*.md5"} );
+        knownConsumers.add( consumer );
+
+        List invalidConsumers = new ArrayList();
+        InvalidScanConsumer badconsumer = new InvalidScanConsumer();
+        invalidConsumers.add( badconsumer );
+
+        RepositoryScanner scanner = lookupRepositoryScanner();
+        RepositoryContentStatistics stats =
+            scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
 
         assertNotNull( "Stats should not be null.", stats );
         assertMinimumHits( "Stats.totalFileCount", 17, stats.getTotalFileCount() );
@@ -172,8 +268,8 @@ public class RepositoryScannerTest
         invalidConsumers.add( badconsumer );
 
         RepositoryScanner scanner = lookupRepositoryScanner();
-        RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers,
+        RepositoryContentStatistics stats =
-                                                          getIgnoreList(), RepositoryScanner.FRESH_SCAN );
+            scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
 
         assertNotNull( "Stats should not be null.", stats );
         assertMinimumHits( "Stats.totalFileCount", actualArtifactPaths.size(), stats.getTotalFileCount() );
@@ -200,7 +296,7 @@ public class RepositoryScannerTest
 
         List knownConsumers = new ArrayList();
         KnownScanConsumer knownConsumer = new KnownScanConsumer();
-        knownConsumer.setIncludes( new String[] { "**/maven-metadata*.xml" } );
+        knownConsumer.setIncludes( new String[]{"**/maven-metadata*.xml"} );
         knownConsumers.add( knownConsumer );
 
         List invalidConsumers = new ArrayList();
@@ -208,8 +304,8 @@ public class RepositoryScannerTest
         invalidConsumers.add( badconsumer );
 
         RepositoryScanner scanner = lookupRepositoryScanner();
-        RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers,
+        RepositoryContentStatistics stats =
-                                                          getIgnoreList(), RepositoryScanner.FRESH_SCAN );
+            scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
 
         assertNotNull( "Stats should not be null.", stats );
         assertMinimumHits( "Stats.totalFileCount", actualMetadataPaths.size(), stats.getTotalFileCount() );
@@ -236,7 +332,7 @@ public class RepositoryScannerTest
 
         List knownConsumers = new ArrayList();
         KnownScanConsumer consumer = new KnownScanConsumer();
-        consumer.setIncludes( new String[] { "**/*.pom" } );
+        consumer.setIncludes( new String[]{"**/*.pom"} );
         knownConsumers.add( consumer );
 
         List invalidConsumers = new ArrayList();
@@ -244,8 +340,8 @@ public class RepositoryScannerTest
         invalidConsumers.add( badconsumer );
 
         RepositoryScanner scanner = lookupRepositoryScanner();
-        RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers,
+        RepositoryContentStatistics stats =
-                                                          getIgnoreList(), RepositoryScanner.FRESH_SCAN );
+            scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
 
         assertNotNull( "Stats should not be null.", stats );
         assertMinimumHits( "Stats.totalFileCount", actualProjectPaths.size(), stats.getTotalFileCount() );
@@ -284,8 +380,8 @@ public class RepositoryScannerTest
         invalidConsumers.add( badconsumer );
 
         RepositoryScanner scanner = lookupRepositoryScanner();
-        RepositoryContentStatistics stats = scanner.scan( repository, knownConsumers, invalidConsumers,
+        RepositoryContentStatistics stats =
-                                                          getIgnoreList(), RepositoryScanner.FRESH_SCAN );
+            scanner.scan( repository, knownConsumers, invalidConsumers, getIgnoreList(), RepositoryScanner.FRESH_SCAN );
 
         assertNotNull( "Stats should not be null.", stats );
         assertMinimumHits( "Stats.totalFileCount", actualArtifactPaths.size(), stats.getTotalFileCount() );