From 40e247c7e41c0550a4a69ebac7f3d70d4fc46d89 Mon Sep 17 00:00:00 2001 From: "Maria Odea B. Ching" Date: Thu, 17 Sep 2009 07:48:55 +0000 Subject: [PATCH] [MRM-1056] Option to force scanning of an artifact/repository regardless of file dates o add checkbox for processing all artifacts beside Scan Repository Now button in repositories page o scan all artifacts if flag is set o added unit tests git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@816084 13f79535-47bb-0310-9956-ffa450edef68 --- .../DefaultRepositoryProxyConnectors.java | 2 +- .../DefaultArchivaTaskScheduler.java | 2 +- .../archiva/scheduled/RepositoryTaskJob.java | 2 +- ...ArchivaRepositoryScanningTaskExecutor.java | 7 +- .../scheduled/tasks/RepositoryTask.java | 12 ++ .../archiva/scheduled/tasks/TaskCreator.java | 7 +- ...ivaRepositoryScanningTaskExecutorTest.java | 127 +++++++++++++++++- .../archiva/web/action/UploadAction.java | 2 +- .../web/action/admin/SchedulerAction.java | 16 ++- .../webapp/WEB-INF/jsp/admin/repositories.jsp | 27 ++-- .../archiva/webdav/ArchivaDavResource.java | 2 +- .../services/AdministrationServiceImpl.java | 2 +- 12 files changed, 179 insertions(+), 29 deletions(-) diff --git a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java index 41fbafa60..5bd96d269 100644 --- a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java +++ b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java @@ -585,7 +585,7 @@ public class DefaultRepositoryProxyConnectors private void queueRepositoryTask( String repositoryId, File localFile ) { - RepositoryTask task = TaskCreator.createRepositoryTask( repositoryId, localFile.getName(), localFile, true ); + RepositoryTask task = TaskCreator.createRepositoryTask( repositoryId, localFile.getName(), localFile, true, true ); try { diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/DefaultArchivaTaskScheduler.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/DefaultArchivaTaskScheduler.java index de956ef8c..5f713197d 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/DefaultArchivaTaskScheduler.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/DefaultArchivaTaskScheduler.java @@ -428,7 +428,7 @@ public class DefaultArchivaTaskScheduler private synchronized void queueInitialRepoScan( ManagedRepositoryConfiguration repoConfig ) { String repoId = repoConfig.getId(); - RepositoryTask task = TaskCreator.createRepositoryTask( repoId, "initial-scan" ); + RepositoryTask task = TaskCreator.createRepositoryTask( repoId, "initial-scan", true ); if ( queuedRepos.contains( repoId ) ) { diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/RepositoryTaskJob.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/RepositoryTaskJob.java index 9088847df..cb9e183bb 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/RepositoryTaskJob.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/RepositoryTaskJob.java @@ -59,7 +59,7 @@ public class RepositoryTaskJob TaskQueue taskQueue = (TaskQueue) dataMap.get( TASK_QUEUE ); String queuePolicy = dataMap.get( TASK_QUEUE_POLICY ).toString(); - RepositoryTask task = TaskCreator.createRepositoryTask( (String) dataMap.get( TASK_REPOSITORY ), "" ); + RepositoryTask task = TaskCreator.createRepositoryTask( (String) dataMap.get( TASK_REPOSITORY ), "", true ); task.setName( context.getJobDetail().getName() ); try diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java index 99ef2fece..916ff4a44 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java @@ -130,11 +130,14 @@ public class ArchivaRepositoryScanningTaskExecutor long sinceWhen = RepositoryScanner.FRESH_SCAN; List results = (List) dao.query( new MostRecentRepositoryScanStatistics( arepo.getId() ) ); - + if ( CollectionUtils.isNotEmpty( results ) ) { RepositoryContentStatistics lastStats = results.get( 0 ); - sinceWhen = lastStats.getWhenGathered().getTime() + lastStats.getDuration(); + if( !repoTask.isScanAll() ) + { + sinceWhen = lastStats.getWhenGathered().getTime() + lastStats.getDuration(); + } } RepositoryScanStatistics stats = repoScanner.scan( arepo, sinceWhen ); diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/RepositoryTask.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/RepositoryTask.java index f20445135..5fc58067d 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/RepositoryTask.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/RepositoryTask.java @@ -42,6 +42,18 @@ public class RepositoryTask boolean updateRelatedArtifacts; + boolean scanAll; + + public boolean isScanAll() + { + return scanAll; + } + + public void setScanAll( boolean scanAll ) + { + this.scanAll = scanAll; + } + public String getRepositoryId() { return repositoryId; diff --git a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java index 60e4c8814..f7bcc5df1 100644 --- a/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java +++ b/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/tasks/TaskCreator.java @@ -31,7 +31,7 @@ import org.apache.maven.archiva.scheduled.DefaultArchivaTaskScheduler; */ public class TaskCreator { - public static RepositoryTask createRepositoryTask( String repositoryId, String taskNameSuffix ) + public static RepositoryTask createRepositoryTask( String repositoryId, String taskNameSuffix, boolean scanAll ) { String suffix = ""; if( !StringUtils.isEmpty( taskNameSuffix ) ) @@ -43,14 +43,15 @@ public class TaskCreator task.setRepositoryId( repositoryId ); task.setName( DefaultArchivaTaskScheduler.REPOSITORY_JOB + ":" + repositoryId + suffix ); task.setQueuePolicy( ArchivaTask.QUEUE_POLICY_WAIT ); + task.setScanAll( scanAll ); return task; } public static RepositoryTask createRepositoryTask( String repositoryId, String taskNameSuffix, File resourceFile, - boolean updateRelatedArtifacts ) + boolean updateRelatedArtifacts, boolean scanAll ) { - RepositoryTask task = createRepositoryTask( repositoryId, taskNameSuffix ); + RepositoryTask task = createRepositoryTask( repositoryId, taskNameSuffix, scanAll ); task.setResourceFile( resourceFile ); task.setUpdateRelatedArtifacts( updateRelatedArtifacts ); diff --git a/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java b/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java index b350982f7..7313bbf22 100644 --- a/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java +++ b/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java @@ -21,6 +21,7 @@ package org.apache.maven.archiva.scheduled.executors; import java.io.File; import java.net.URL; +import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.Properties; @@ -34,11 +35,13 @@ import org.apache.maven.archiva.database.ArchivaDAO; import org.apache.maven.archiva.database.ArtifactDAO; import org.apache.maven.archiva.database.constraints.ArtifactsProcessedConstraint; import org.apache.maven.archiva.model.ArchivaArtifact; +import org.apache.maven.archiva.model.RepositoryContentStatistics; import org.apache.maven.archiva.scheduled.tasks.RepositoryTask; import org.codehaus.plexus.jdo.DefaultConfigurableJdoFactory; import org.codehaus.plexus.jdo.JdoFactory; import org.codehaus.plexus.spring.PlexusInSpringTestCase; import org.codehaus.plexus.taskqueue.execution.TaskExecutor; +import org.codehaus.plexus.util.FileUtils; import org.jpox.SchemaTool; /** @@ -53,6 +56,8 @@ public class ArchivaRepositoryScanningTaskExecutorTest protected ArchivaDAO dao; + private File repoDir; + protected void setUp() throws Exception { @@ -131,12 +136,28 @@ public class ArchivaRepositoryScanningTaskExecutorTest this.dao = (ArchivaDAO) lookup( ArchivaDAO.class.getName(), "jdo" ); taskExecutor = (TaskExecutor) lookup( TaskExecutor.class, "test-repository-scanning" ); + + File sourceRepoDir = new File( getBasedir(), "src/test/repositories/default-repository" ); + repoDir = new File( getBasedir(), "target/default-repository" ); + + repoDir.mkdir(); + + FileUtils.copyDirectoryStructure( sourceRepoDir, repoDir ); + + assertTrue( repoDir.exists() ); } - + + protected void tearDown() throws Exception + { + FileUtils.deleteDirectory( repoDir ); + + assertFalse( repoDir.exists() ); + + super.tearDown(); + } + public void testExecutor() throws Exception { - File repoDir = new File( getBasedir(), "src/test/repositories/default-repository" ); - assertTrue( "Default Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() ); ArchivaConfiguration archivaConfig = (ArchivaConfiguration) lookup( ArchivaConfiguration.class ); @@ -162,6 +183,106 @@ public class ArchivaRepositoryScanningTaskExecutorTest assertEquals("Incorrect number of unprocessed artifacts detected.", 8, unprocessedResultList.size() ); } + public void testExecutorScanOnlyNewArtifacts() + throws Exception + { + assertTrue( "Default Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() ); + + ArchivaConfiguration archivaConfig = (ArchivaConfiguration) lookup( ArchivaConfiguration.class ); + assertNotNull( archivaConfig ); + + // Create it + ManagedRepositoryConfiguration repo = createRepository( "testRepo", "Test Repository", repoDir ); + assertNotNull( repo ); + archivaConfig.getConfiguration().getManagedRepositories().clear(); + archivaConfig.getConfiguration().addManagedRepository( repo ); + + RepositoryTask repoTask = new RepositoryTask(); + + repoTask.setName( "testRepoTask" ); + repoTask.setRepositoryId( "testRepo" ); + repoTask.setScanAll( false ); + + RepositoryContentStatistics stats = new RepositoryContentStatistics(); + stats.setDuration( 1234567 ); + stats.setNewFileCount( 8 ); + stats.setRepositoryId( "testRepo" ); + stats.setTotalArtifactCount( 8 ); + stats.setTotalFileCount( 8 ); + stats.setTotalGroupCount( 3 ); + stats.setTotalProjectCount( 5 ); + stats.setTotalSize( 999999 ); + stats.setWhenGathered( Calendar.getInstance().getTime() ); + + dao.getRepositoryContentStatisticsDAO().saveRepositoryContentStatistics( stats ); + + taskExecutor.executeTask( repoTask ); + + ArtifactDAO adao = dao.getArtifactDAO(); + List unprocessedResultList = adao.queryArtifacts( new ArtifactsProcessedConstraint( false ) ); + + assertNotNull( unprocessedResultList ); + assertEquals("Incorrect number of unprocessed artifacts detected. No new artifacts should have been found.", 0, unprocessedResultList.size() ); + + File newArtifactGroup = new File( repoDir, "org/apache/archiva"); + + FileUtils.copyDirectoryStructure( new File( getBasedir(), "target/test-classes/test-repo/org/apache/archiva"), newArtifactGroup ); + + // update last modified date + new File( newArtifactGroup, "archiva-index-methods-jar-test/1.0/pom.xml" ).setLastModified( Calendar.getInstance().getTimeInMillis() + 1000 ); + new File( newArtifactGroup, "archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ).setLastModified( Calendar.getInstance().getTimeInMillis() + 1000 ); + + assertTrue( newArtifactGroup.exists() ); + + taskExecutor.executeTask( repoTask ); + + unprocessedResultList = adao.queryArtifacts( new ArtifactsProcessedConstraint( false ) ); + assertNotNull( unprocessedResultList ); + assertEquals( "Incorrect number of unprocessed artifacts detected. One new artifact should have been found.", 1, unprocessedResultList.size() ); + } + + public void testExecutorForceScanAll() + throws Exception + { + assertTrue( "Default Test Repository should exist.", repoDir.exists() && repoDir.isDirectory() ); + + ArchivaConfiguration archivaConfig = (ArchivaConfiguration) lookup( ArchivaConfiguration.class ); + assertNotNull( archivaConfig ); + + // Create it + ManagedRepositoryConfiguration repo = createRepository( "testRepo", "Test Repository", repoDir ); + assertNotNull( repo ); + archivaConfig.getConfiguration().getManagedRepositories().clear(); + archivaConfig.getConfiguration().addManagedRepository( repo ); + + RepositoryTask repoTask = new RepositoryTask(); + + repoTask.setName( "testRepoTask" ); + repoTask.setRepositoryId( "testRepo" ); + repoTask.setScanAll( true ); + + RepositoryContentStatistics stats = new RepositoryContentStatistics(); + stats.setDuration( 1234567 ); + stats.setNewFileCount( 8 ); + stats.setRepositoryId( "testRepo" ); + stats.setTotalArtifactCount( 8 ); + stats.setTotalFileCount( 8 ); + stats.setTotalGroupCount( 3 ); + stats.setTotalProjectCount( 5 ); + stats.setTotalSize( 999999 ); + stats.setWhenGathered( Calendar.getInstance().getTime() ); + + dao.getRepositoryContentStatisticsDAO().saveRepositoryContentStatistics( stats ); + + taskExecutor.executeTask( repoTask ); + + ArtifactDAO adao = dao.getArtifactDAO(); + List unprocessedResultList = adao.queryArtifacts( new ArtifactsProcessedConstraint( false ) ); + + assertNotNull( unprocessedResultList ); + assertEquals("Incorrect number of unprocessed artifacts detected.", 8, unprocessedResultList.size() ); + } + protected ManagedRepositoryConfiguration createRepository( String id, String name, File location ) { ManagedRepositoryConfiguration repo = new ManagedRepositoryConfiguration(); diff --git a/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/UploadAction.java b/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/UploadAction.java index e64a46d0c..9c538c837 100644 --- a/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/UploadAction.java +++ b/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/UploadAction.java @@ -617,7 +617,7 @@ public class UploadAction private void queueRepositoryTask( String repositoryId, File localFile ) { - RepositoryTask task = TaskCreator.createRepositoryTask( repositoryId, localFile.getName(), localFile, true ); + RepositoryTask task = TaskCreator.createRepositoryTask( repositoryId, localFile.getName(), localFile, true, true ); try { diff --git a/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/SchedulerAction.java b/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/SchedulerAction.java index 25a9bb510..711a553cf 100644 --- a/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/SchedulerAction.java +++ b/archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/maven/archiva/web/action/admin/SchedulerAction.java @@ -50,7 +50,9 @@ public class SchedulerAction private ArchivaTaskScheduler taskScheduler; private String repoid; - + + private boolean scanAll; + public String scanRepository() { if ( StringUtils.isBlank( repoid ) ) @@ -59,7 +61,7 @@ public class SchedulerAction return SUCCESS; } - RepositoryTask task = TaskCreator.createRepositoryTask( repoid, "" ); + RepositoryTask task = TaskCreator.createRepositoryTask( repoid, "", scanAll ); boolean scheduleTask = false; @@ -181,4 +183,14 @@ public class SchedulerAction { this.repoid = repoid; } + + public boolean getScanAll() + { + return scanAll; + } + + public void setScanAll( boolean scanAll ) + { + this.scanAll = scanAll; + } } diff --git a/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositories.jsp b/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositories.jsp index e73cc3711..22d56a94b 100644 --- a/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositories.jsp +++ b/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/repositories.jsp @@ -198,20 +198,21 @@ Actions - - - - -
- - - - - - -
+ + + + + + + + + + +
Process All Artifacts
+ + - + Stats diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/maven/archiva/webdav/ArchivaDavResource.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/maven/archiva/webdav/ArchivaDavResource.java index c0b5ec496..493d17d71 100644 --- a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/maven/archiva/webdav/ArchivaDavResource.java +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/maven/archiva/webdav/ArchivaDavResource.java @@ -645,7 +645,7 @@ public class ArchivaDavResource private void queueRepositoryTask( File localFile ) { - RepositoryTask task = TaskCreator.createRepositoryTask( repository.getId(), localFile.getName(), localFile, false ); + RepositoryTask task = TaskCreator.createRepositoryTask( repository.getId(), localFile.getName(), localFile, false, true ); try { diff --git a/archiva-modules/archiva-web/archiva-xmlrpc/archiva-xmlrpc-services/src/main/java/org/apache/archiva/web/xmlrpc/services/AdministrationServiceImpl.java b/archiva-modules/archiva-web/archiva-xmlrpc/archiva-xmlrpc-services/src/main/java/org/apache/archiva/web/xmlrpc/services/AdministrationServiceImpl.java index 623b32475..3279c79c7 100644 --- a/archiva-modules/archiva-web/archiva-xmlrpc/archiva-xmlrpc-services/src/main/java/org/apache/archiva/web/xmlrpc/services/AdministrationServiceImpl.java +++ b/archiva-modules/archiva-web/archiva-xmlrpc/archiva-xmlrpc-services/src/main/java/org/apache/archiva/web/xmlrpc/services/AdministrationServiceImpl.java @@ -313,7 +313,7 @@ public class AdministrationServiceImpl } } - RepositoryTask task = TaskCreator.createRepositoryTask( repoId, "" ); + RepositoryTask task = TaskCreator.createRepositoryTask( repoId, "", false ); taskScheduler.queueRepositoryTask( task );