Apache
/
maven
mirror of https://github.com/apache/maven.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[MNG-4432] reimplement parallel artifacts download 

git-svn-id: https://svn.apache.org/repos/asf/maven/maven-3/trunk@900982 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Benjamin Bentmann 2010-01-19 22:25:12 +00:00
parent 34c2aa8acc
commit 9417640359
3 changed files with 225 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java
View File

@ -24,6 +24,12 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.factory.ArtifactFactory;
@ -97,6 +103,38 @@ public class DefaultArtifactResolver
@Requirement
private LegacySupport legacySupport;
private final Executor executor;
public DefaultArtifactResolver()
{
int threads = Integer.getInteger( "maven.artifact.threads", 5 ).intValue();
if ( threads <= 1 )
{
executor = new Executor()
{
public void execute( Runnable command )
{
command.run();
}
};
}
else
{
executor =
new ThreadPoolExecutor( threads, threads, 3, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>() );
}
}
@Override
protected void finalize()
throws Throwable
{
if ( executor instanceof ExecutorService )
{
( (ExecutorService) executor ).shutdown();
}
}
private void injectSession( RepositoryRequest request )
{
MavenSession session = legacySupport.getSession();
@ -558,41 +596,41 @@ public class DefaultArtifactResolver
{
return result;
}
if ( result.getArtifactResolutionNodes() != null )
{
ArtifactResolutionRequest childRequest = new ArtifactResolutionRequest( request );
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
CountDownLatch latch = new CountDownLatch( result.getArtifactResolutionNodes().size() );
for ( ResolutionNode node : result.getArtifactResolutionNodes() )
{
Artifact artifact = node.getArtifact();
try
if ( resolutionFilter == null || resolutionFilter.include( artifact ) )
{
if ( resolutionFilter == null || resolutionFilter.include( artifact ) )
{
childRequest.setRemoteRepositories( node.getRemoteRepositories() );
ArtifactResolutionRequest childRequest = new ArtifactResolutionRequest( request );
childRequest.setRemoteRepositories( node.getRemoteRepositories() );
resolve( artifact, childRequest, transferListener, false );
}
executor.execute( new ResolveTask( classLoader, latch, artifact, transferListener, childRequest,
result ) );
}
catch ( ArtifactNotFoundException anfe )
else
{
// These are cases where the artifact just isn't present in any of the remote repositories
// because it wasn't deployed, or it was deployed in the wrong place.
result.addMissingArtifact( artifact );
}
catch ( ArtifactResolutionException e )
{
// This is really a wagon TransferFailedException so something went wrong after we successfully
// retrieved the metadata.
result.addErrorArtifactException( e );
latch.countDown();
}
}
try
{
latch.await();
}
catch ( InterruptedException e )
{
result.addErrorArtifactException( new ArtifactResolutionException( "Resolution interrupted",
rootArtifact, e ) );
}
}
// We want to send the root artifact back in the result but we need to do this after the other dependencies
// have been resolved.
if ( request.isResolveRoot() )
@ -612,4 +650,67 @@ public class DefaultArtifactResolver
{
resolve( artifact, remoteRepositories, localRepository, null );
}
private class ResolveTask
implements Runnable
{
private final ClassLoader classLoader;
private final CountDownLatch latch;
private final Artifact artifact;
private final TransferListener transferListener;
private final ArtifactResolutionRequest request;
private final ArtifactResolutionResult result;
public ResolveTask( ClassLoader classLoader, CountDownLatch latch, Artifact artifact, TransferListener transferListener,
ArtifactResolutionRequest request, ArtifactResolutionResult result )
{
this.classLoader = classLoader;
this.latch = latch;
this.artifact = artifact;
this.transferListener = transferListener;
this.request = request;
this.result = result;
}
public void run()
{
try
{
Thread.currentThread().setContextClassLoader( classLoader );
resolve( artifact, request, transferListener, false );
}
catch ( ArtifactNotFoundException anfe )
{
// These are cases where the artifact just isn't present in any of the remote repositories
// because it wasn't deployed, or it was deployed in the wrong place.
synchronized ( result )
{
result.addMissingArtifact( artifact );
}
}
catch ( ArtifactResolutionException e )
{
// This is really a wagon TransferFailedException so something went wrong after we successfully
// retrieved the metadata.
synchronized ( result )
{
result.addErrorArtifactException( e );
}
}
finally
{
latch.countDown();
}
}
}
}

67
maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java
View File

@ -34,11 +34,11 @@ public class TransferListenerAdapter
implements TransferListener
{
private ArtifactTransferListener listener;
private final ArtifactTransferListener listener;
private Map<Resource, ArtifactTransferResource> artifacts;
private final Map<Resource, ArtifactTransferResource> artifacts;
private Map<Resource, Long> transfers;
private final Map<Resource, Long> transfers;
public static TransferListener newAdapter( ArtifactTransferListener listener )
{
@ -67,22 +67,34 @@ public class TransferListenerAdapter
{
ArtifactTransferEvent event = wrap( transferEvent );
Long transferred = transfers.get( transferEvent.getResource() );
Long transferred = null;
synchronized ( transfers )
{
transferred = transfers.remove( transferEvent.getResource() );
}
if ( transferred != null )
{
event.setTransferredBytes( transferred.longValue() );
}
listener.transferCompleted( event );
synchronized ( artifacts )
{
artifacts.remove( transferEvent.getResource() );
}
artifacts.remove( transferEvent.getResource() );
transfers.remove( transferEvent.getResource() );
listener.transferCompleted( event );
}
public void transferError( TransferEvent transferEvent )
{
artifacts.remove( transferEvent.getResource() );
transfers.remove( transferEvent.getResource() );
synchronized ( transfers )
{
transfers.remove( transferEvent.getResource() );
}
synchronized ( artifacts )
{
artifacts.remove( transferEvent.getResource() );
}
}
public void transferInitiated( TransferEvent transferEvent )
@ -92,16 +104,20 @@ public class TransferListenerAdapter
public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
{
Long transferred = transfers.get( transferEvent.getResource() );
if ( transferred == null )
Long transferred;
synchronized ( transfers )
{
transferred = Long.valueOf( length );
transferred = transfers.get( transferEvent.getResource() );
if ( transferred == null )
{
transferred = Long.valueOf( length );
}
else
{
transferred = Long.valueOf( transferred.longValue() + length );
}
transfers.put( transferEvent.getResource(), transferred );
}
else
{
transferred = Long.valueOf( transferred.longValue() + length );
}
transfers.put( transferEvent.getResource(), transferred );
ArtifactTransferEvent event = wrap( transferEvent );
event.setDataBuffer( buffer );
@ -153,15 +169,18 @@ public class TransferListenerAdapter
}
else
{
ArtifactTransferResource artifact = artifacts.get( resource );
if ( artifact == null )
synchronized ( artifacts )
{
artifact = new MavenArtifact( repository.getUrl(), resource );
artifacts.put( resource, artifact );
}
ArtifactTransferResource artifact = artifacts.get( resource );
return artifact;
if ( artifact == null )
{
artifact = new MavenArtifact( repository.getUrl(), resource );
artifacts.put( resource, artifact );
}
return artifact;
}
}
}

68
maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java
View File

@ -20,18 +20,27 @@ package org.apache.maven.cli;
*/
import java.io.PrintStream;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.maven.repository.ArtifactTransferEvent;
import org.apache.maven.repository.ArtifactTransferResource;
/**
* Console download progress meter.
*
*
* @author <a href="mailto:brett@apache.org">Brett Porter</a>
*/
class ConsoleMavenTransferListener
extends AbstractMavenTransferListener
{
private Map<ArtifactTransferResource, Long> downloads =
Collections.synchronizedMap( new LinkedHashMap<ArtifactTransferResource, Long>() );
private int lastLength;
public ConsoleMavenTransferListener( PrintStream out )
{
super( out );
@ -40,26 +49,69 @@ class ConsoleMavenTransferListener
@Override
protected void doProgress( ArtifactTransferEvent transferEvent )
{
long total = transferEvent.getResource().getContentLength();
long complete = transferEvent.getTransferredBytes();
ArtifactTransferResource resource = transferEvent.getResource();
downloads.put( resource, Long.valueOf( transferEvent.getTransferredBytes() ) );
// TODO [BP]: Sys.out may no longer be appropriate, but will \r work with getLogger()?
StringBuilder buffer = new StringBuilder( 64 );
for ( Map.Entry<ArtifactTransferResource, Long> entry : downloads.entrySet() )
{
long total = entry.getKey().getContentLength();
long complete = entry.getValue().longValue();
buffer.append( getStatus( complete, total ) ).append( " " );
}
int pad = lastLength - buffer.length();
lastLength = buffer.length();
pad( buffer, pad );
buffer.append( '\r' );
out.print( buffer );
}
private String getStatus( long complete, long total )
{
if ( total >= 1024 )
{
out.print( toKB( complete ) + "/" + toKB( total ) + " KB " + "\r" );
return toKB( complete ) + "/" + toKB( total ) + " KB ";
}
else if ( total >= 0 )
{
out.print( complete + "/" + total + " B " + "\r" );
return complete + "/" + total + " B ";
}
else if ( complete >= 1024 )
{
out.print( toKB( complete ) + " KB " + "\r" );
return toKB( complete ) + " KB ";
}
else
{
out.print( complete + " B " + "\r" );
return complete + " B ";
}
}
private void pad( StringBuilder buffer, int spaces )
{
String block = " ";
while ( spaces > 0 )
{
int n = Math.min( spaces, block.length() );
buffer.append( block, 0, n );
spaces -= n;
}
}
@Override
public void transferCompleted( ArtifactTransferEvent transferEvent )
{
downloads.remove( transferEvent.getResource() );
StringBuilder buffer = new StringBuilder( 64 );
pad( buffer, lastLength );
buffer.append( '\r' );
out.print( buffer );
super.transferCompleted( transferEvent );
}
}