Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix unit tests in archiva-scheduler-indexing 

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1197140 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Olivier Lamy 2011-11-03 14:30:03 +00:00
parent 42edbc5ca5
commit c215c38616
3 changed files with 155 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java
View File

@ -175,8 +175,8 @@ public class DefaultDownloadRemoteIndexScheduler
DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest = DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest =
new DownloadRemoteIndexTaskRequest().setRemoteRepository( remoteRepository ).setNetworkProxy( new DownloadRemoteIndexTaskRequest().setRemoteRepository( remoteRepository ).setNetworkProxy(
networkProxy ).setFullDownload( fullDownload ).setWagonFactory( wagonFactory ).setNexusIndexer( networkProxy ).setFullDownload( fullDownload ).setWagonFactory(
nexusIndexer ).setIndexUpdater( indexUpdater ); wagonFactory ).setRemoteRepositoryAdmin( remoteRepositoryAdmin ).setIndexUpdater( indexUpdater );
if ( now ) if ( now )
{ {

259
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java
View File

@ -18,13 +18,14 @@ package org.apache.archiva.scheduler.indexing;
* under the License. * under the License.
*/ */
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.NetworkProxy; import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.RemoteRepository; import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.proxy.common.WagonFactory; import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.archiva.proxy.common.WagonFactoryException; import org.apache.archiva.proxy.common.WagonFactoryException;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.time.StopWatch; import org.apache.commons.lang.time.StopWatch;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.updater.IndexUpdateRequest; import org.apache.maven.index.updater.IndexUpdateRequest;
import org.apache.maven.index.updater.IndexUpdater; import org.apache.maven.index.updater.IndexUpdater;
@ -63,11 +64,11 @@ import java.util.List;
public class DownloadRemoteIndexTask public class DownloadRemoteIndexTask
implements Runnable implements Runnable
{ {
private Logger log = LoggerFactory.getLogger( getClass( ) ); private Logger log = LoggerFactory.getLogger( getClass() );
private RemoteRepository remoteRepository; private RemoteRepository remoteRepository;
private NexusIndexer nexusIndexer; private RemoteRepositoryAdmin remoteRepositoryAdmin;
private WagonFactory wagonFactory; private WagonFactory wagonFactory;
@ -82,166 +83,126 @@ public class DownloadRemoteIndexTask
public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest, public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
List<String> runningRemoteDownloadIds ) List<String> runningRemoteDownloadIds )
{ {
this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository( ); this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
this.nexusIndexer = downloadRemoteIndexTaskRequest.getNexusIndexer( ); this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory( ); this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy( ); this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload( );
this.runningRemoteDownloadIds = runningRemoteDownloadIds; this.runningRemoteDownloadIds = runningRemoteDownloadIds;
this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater( ); this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
} }
public void run( ) public void run()
{ {
// so short lock : not sure we need it // so short lock : not sure we need it
synchronized ( this.runningRemoteDownloadIds ) synchronized ( this.runningRemoteDownloadIds )
{ {
if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId( ) ) ) if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
{ {
// skip it as it's running // skip it as it's running
log.info( "skip download index remote for repo {} it's already running", log.info( "skip download index remote for repo {} it's already running",
this.remoteRepository.getId( ) ); this.remoteRepository.getId() );
return; return;
} }
this.runningRemoteDownloadIds.add( this.remoteRepository.getId( ) ); this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
} }
StopWatch stopWatch = new StopWatch( ); File tempIndexDirectory = null;
stopWatch.start( ); StopWatch stopWatch = new StopWatch();
log.info( "start download remote index for remote repository " + this.remoteRepository.getId( ) ); stopWatch.start();
IndexingContext indexingContext =
nexusIndexer.getIndexingContexts( ).get( "remote-" + remoteRepository.getId( ) );
// TODO check if null ? normally not as created by DefaultDownloadRemoteIndexScheduler#startup
// create a temp directory to download files
final File tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" );
File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" );
indexCacheDirectory.mkdirs();
try try
{ {
if ( tempIndexDirectory.exists( ) ) log.info( "start download remote index for remote repository " + this.remoteRepository.getId() );
IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
// create a temp directory to download files
tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
indexCacheDirectory.mkdirs();
if ( tempIndexDirectory.exists() )
{ {
FileUtils.deleteDirectory( tempIndexDirectory ); FileUtils.deleteDirectory( tempIndexDirectory );
} }
tempIndexDirectory.mkdirs( ); tempIndexDirectory.mkdirs();
tempIndexDirectory.deleteOnExit(); tempIndexDirectory.deleteOnExit();
String baseIndexUrl = indexingContext.getIndexUpdateUrl( ); String baseIndexUrl = indexingContext.getIndexUpdateUrl();
final Wagon wagon = wagonFactory.getWagon( new URL( this.remoteRepository.getUrl( ) ).getProtocol( ) ); final Wagon wagon = wagonFactory.getWagon( new URL( this.remoteRepository.getUrl() ).getProtocol() );
setupWagonReadTimeout( wagon ); setupWagonReadTimeout( wagon );
// TODO transferListener // TODO transferListener
wagon.addTransferListener( new DownloadListener( ) ); wagon.addTransferListener( new DownloadListener() );
ProxyInfo proxyInfo = null; ProxyInfo proxyInfo = null;
if ( this.networkProxy != null ) if ( this.networkProxy != null )
{ {
proxyInfo = new ProxyInfo( ); proxyInfo = new ProxyInfo();
proxyInfo.setHost( this.networkProxy.getHost( ) ); proxyInfo.setHost( this.networkProxy.getHost() );
proxyInfo.setPort( this.networkProxy.getPort( ) ); proxyInfo.setPort( this.networkProxy.getPort() );
proxyInfo.setUserName( this.networkProxy.getUsername( ) ); proxyInfo.setUserName( this.networkProxy.getUsername() );
proxyInfo.setPassword( this.networkProxy.getPassword( ) ); proxyInfo.setPassword( this.networkProxy.getPassword() );
} }
AuthenticationInfo authenticationInfo = null; AuthenticationInfo authenticationInfo = null;
if ( this.remoteRepository.getUserName( ) != null ) if ( this.remoteRepository.getUserName() != null )
{ {
authenticationInfo = new AuthenticationInfo( ); authenticationInfo = new AuthenticationInfo();
authenticationInfo.setUserName( this.remoteRepository.getUserName( ) ); authenticationInfo.setUserName( this.remoteRepository.getUserName() );
authenticationInfo.setPassword( this.remoteRepository.getPassword( ) ); authenticationInfo.setPassword( this.remoteRepository.getPassword() );
} }
wagon.connect( new Repository( this.remoteRepository.getId( ), baseIndexUrl ), authenticationInfo, wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
proxyInfo ); proxyInfo );
File indexDirectory = indexingContext.getIndexDirectoryFile( ); File indexDirectory = indexingContext.getIndexDirectoryFile();
if ( !indexDirectory.exists( ) ) if ( !indexDirectory.exists() )
{ {
indexDirectory.mkdirs( ); indexDirectory.mkdirs();
} }
ResourceFetcher resourceFetcher = new ResourceFetcher( ) ResourceFetcher resourceFetcher = new WagonResourceFetcher( log, tempIndexDirectory, wagon );
{
public void connect( String id, String url )
throws IOException
{
//no op
}
public void disconnect( )
throws IOException
{
// no op
}
public InputStream retrieve( String name )
throws IOException, FileNotFoundException
{
try
{
log.info( "index update retrieve file, name:{}", name );
File file = new File( tempIndexDirectory, name );
if ( file.exists( ) )
{
file.delete( );
}
file.deleteOnExit();
wagon.get( name, file );
return new FileInputStream( file );
}
catch ( AuthorizationException e )
{
throw new IOException( e.getMessage( ) );
}
catch ( TransferFailedException e )
{
throw new IOException( e.getMessage( ) );
}
catch ( ResourceDoesNotExistException e )
{
throw new FileNotFoundException( e.getMessage( ) );
}
}
};
IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher ); IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
request.setForceFullUpdate( this.fullDownload ); request.setForceFullUpdate( this.fullDownload );
request.setLocalIndexCacheDir( indexCacheDirectory ); request.setLocalIndexCacheDir( indexCacheDirectory );
this.indexUpdater.fetchAndUpdateIndex( request ); this.indexUpdater.fetchAndUpdateIndex( request );
stopWatch.stop( ); stopWatch.stop();
log.info( "time to download remote repository index for repository {}: {} s", log.info( "time to download remote repository index for repository {}: {} s", this.remoteRepository.getId(),
this.remoteRepository.getId( ), ( stopWatch.getTime( ) / 1000 ) ); ( stopWatch.getTime() / 1000 ) );
} }
catch ( MalformedURLException e ) catch ( MalformedURLException e )
{ {
log.error( e.getMessage( ), e ); log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage( ), e ); throw new RuntimeException( e.getMessage(), e );
} }
catch ( WagonFactoryException e ) catch ( WagonFactoryException e )
{ {
log.error( e.getMessage( ), e ); log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage( ), e ); throw new RuntimeException( e.getMessage(), e );
} }
catch ( ConnectionException e ) catch ( ConnectionException e )
{ {
log.error( e.getMessage( ), e ); log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage( ), e ); throw new RuntimeException( e.getMessage(), e );
} }
catch ( AuthenticationException e ) catch ( AuthenticationException e )
{ {
log.error( e.getMessage( ), e ); log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage( ), e ); throw new RuntimeException( e.getMessage(), e );
} }
catch ( IOException e ) catch ( IOException e )
{ {
log.error( e.getMessage( ), e ); log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage( ), e ); throw new RuntimeException( e.getMessage(), e );
}
catch ( RepositoryAdminException e )
{
log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage(), e );
} }
finally finally
{ {
deleteDirectoryQuiet( tempIndexDirectory ); deleteDirectoryQuiet( tempIndexDirectory );
this.runningRemoteDownloadIds.remove( this.remoteRepository.getId( ) ); this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
} }
log.info( "end download remote index for remote repository " + this.remoteRepository.getId( ) ); log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
} }
private void deleteDirectoryQuiet( File f ) private void deleteDirectoryQuiet( File f )
@ -252,7 +213,7 @@ public class DownloadRemoteIndexTask
} }
catch ( IOException e ) catch ( IOException e )
{ {
log.warn( "skip error delete " + f + ": " + e.getMessage( ) ); log.warn( "skip error delete " + f + ": " + e.getMessage() );
} }
} }
@ -260,15 +221,15 @@ public class DownloadRemoteIndexTask
{ {
try try
{ {
HttpConfiguration httpConfiguration = new HttpConfiguration( ).setAll( HttpConfiguration httpConfiguration = new HttpConfiguration().setAll(
new HttpMethodConfiguration( ).setReadTimeout( remoteRepository.getRemoteDownloadTimeout( ) * 1000 ) ); new HttpMethodConfiguration().setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 ) );
Method setHttpConfigurationMethod = Method setHttpConfigurationMethod =
wagon.getClass( ).getMethod( "setHttpConfiguration", HttpConfiguration.class ); wagon.getClass().getMethod( "setHttpConfiguration", HttpConfiguration.class );
setHttpConfigurationMethod.invoke( wagon, httpConfiguration ); setHttpConfigurationMethod.invoke( wagon, httpConfiguration );
} }
catch ( Exception e ) catch ( Exception e )
{ {
log.debug( "unable to set download remote time out for index {}", e.getMessage( ), e ); log.debug( "unable to set download remote time out for index {}", e.getMessage(), e );
} }
} }
@ -276,7 +237,7 @@ public class DownloadRemoteIndexTask
public static class DownloadListener public static class DownloadListener
implements TransferListener implements TransferListener
{ {
private Logger log = LoggerFactory.getLogger( getClass( ) ); private Logger log = LoggerFactory.getLogger( getClass() );
String reourceName; String reourceName;
@ -284,36 +245,36 @@ public class DownloadRemoteIndexTask
public void transferInitiated( TransferEvent transferEvent ) public void transferInitiated( TransferEvent transferEvent )
{ {
reourceName = transferEvent.getResource( ).getName( ); reourceName = transferEvent.getResource().getName();
log.debug( "initiate transfer of {}", reourceName ); log.debug( "initiate transfer of {}", reourceName );
} }
public void transferStarted( TransferEvent transferEvent ) public void transferStarted( TransferEvent transferEvent )
{ {
reourceName = transferEvent.getResource( ).getName( ); reourceName = transferEvent.getResource().getName();
startTime = System.currentTimeMillis( ); startTime = System.currentTimeMillis();
log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) ); log.info( "start transfer of {}", transferEvent.getResource().getName() );
} }
public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length ) public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
{ {
log.debug( "transfer of {} : {}/{}", log.debug( "transfer of {} : {}/{}",
Arrays.asList( transferEvent.getResource( ).getName( ), buffer.length, length ).toArray( ) ); Arrays.asList( transferEvent.getResource().getName(), buffer.length, length ).toArray() );
} }
public void transferCompleted( TransferEvent transferEvent ) public void transferCompleted( TransferEvent transferEvent )
{ {
reourceName = transferEvent.getResource( ).getName( ); reourceName = transferEvent.getResource().getName();
long endTime = System.currentTimeMillis( ); long endTime = System.currentTimeMillis();
log.info( "end of transfer file {}: {}s", transferEvent.getResource( ).getName( ), log.info( "end of transfer file {}: {}s", transferEvent.getResource().getName(),
( endTime - startTime ) / 1000 ); ( endTime - startTime ) / 1000 );
} }
public void transferError( TransferEvent transferEvent ) public void transferError( TransferEvent transferEvent )
{ {
log.info( "error of transfer file {}: {}", Arrays.asList( transferEvent.getResource( ).getName( ), log.info( "error of transfer file {}: {}", Arrays.asList( transferEvent.getResource().getName(),
transferEvent.getException( ).getMessage( ) ).toArray( transferEvent.getException().getMessage() ).toArray(
new Object[2] ), transferEvent.getException( ) ); new Object[2] ), transferEvent.getException() );
} }
public void debug( String message ) public void debug( String message )
@ -322,4 +283,64 @@ public class DownloadRemoteIndexTask
} }
} }
private static class WagonResourceFetcher
implements ResourceFetcher
{
Logger log;
File tempIndexDirectory;
Wagon wagon;
private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon )
{
this.log = log;
this.tempIndexDirectory = tempIndexDirectory;
this.wagon = wagon;
}
public void connect( String id, String url )
throws IOException
{
//no op
}
public void disconnect()
throws IOException
{
// no op
}
public InputStream retrieve( String name )
throws IOException, FileNotFoundException
{
try
{
log.info( "index update retrieve file, name:{}", name );
File file = new File( tempIndexDirectory, name );
if ( file.exists() )
{
file.delete();
}
file.deleteOnExit();
wagon.get( name, file );
return new FileInputStream( file );
}
catch ( AuthorizationException e )
{
throw new IOException( e.getMessage() );
}
catch ( TransferFailedException e )
{
throw new IOException( e.getMessage() );
}
catch ( ResourceDoesNotExistException e )
{
throw new FileNotFoundException( e.getMessage() );
}
}
}
} }

24
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java
View File

@ -20,6 +20,7 @@ package org.apache.archiva.scheduler.indexing;
import org.apache.archiva.admin.model.beans.NetworkProxy; import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.RemoteRepository; import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.proxy.common.WagonFactory; import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.maven.index.NexusIndexer; import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.updater.IndexUpdater; import org.apache.maven.index.updater.IndexUpdater;
@ -32,7 +33,7 @@ public class DownloadRemoteIndexTaskRequest
{ {
private RemoteRepository remoteRepository; private RemoteRepository remoteRepository;
private NexusIndexer nexusIndexer; private RemoteRepositoryAdmin remoteRepositoryAdmin;
private WagonFactory wagonFactory; private WagonFactory wagonFactory;
@ -58,16 +59,6 @@ public class DownloadRemoteIndexTaskRequest
return this; return this;
} }
public NexusIndexer getNexusIndexer()
{
return nexusIndexer;
}
public DownloadRemoteIndexTaskRequest setNexusIndexer( NexusIndexer nexusIndexer )
{
this.nexusIndexer = nexusIndexer;
return this;
}
public WagonFactory getWagonFactory() public WagonFactory getWagonFactory()
{ {
@ -112,4 +103,15 @@ public class DownloadRemoteIndexTaskRequest
this.indexUpdater = indexUpdater; this.indexUpdater = indexUpdater;
return this; return this;
} }
public RemoteRepositoryAdmin getRemoteRepositoryAdmin()
{
return remoteRepositoryAdmin;
}
public DownloadRemoteIndexTaskRequest setRemoteRepositoryAdmin( RemoteRepositoryAdmin remoteRepositoryAdmin )
{
this.remoteRepositoryAdmin = remoteRepositoryAdmin;
return this;
}
} }