Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding packedIndexDir

This commit is contained in:
Martin Stockhammer 2018-03-29 23:47:13 +02:00
parent f18aa86c68
commit 9cd3c61340
19 changed files with 194 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo
View File

@ -470,6 +470,15 @@
The directory for the indexes of this repository.
</description>
</field>
<field>
<name>packedIndexDir</name>
<version>3.0.0+</version>
<type>String</type>
<defaultValue></defaultValue>
<description>
The directory for the packed indexes of this repository.
</description>
</field>
<field>
<name>description</name>
<version>1.0.0+</version>

31
archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java
View File

@ -123,10 +123,9 @@ public class MavenIndexManager implements ArchivaIndexManager {
@Inject
private ArtifactContextProducer artifactContextProducer;
@Inject
RepositoryRegistry repositoryRegistry;
public static final String DEFAULT_INDEXER_DIR = ".indexer";
public static final String DEFAULT_PACKED_INDEX_DIR = ".index";
private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( );
@ -523,6 +522,7 @@ public class MavenIndexManager implements ArchivaIndexManager {
IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
try {
icf.setLocalIndexPath(getIndexPath(repo));
icf.setLocalPackedIndexPath(getPackedIndexPath(repo));
} catch (IOException e) {
log.error("Could not set local index path for {}. New URI: {}", repo.getId(), icf.getIndexPath());
}
@ -556,6 +556,33 @@ public class MavenIndexManager implements ArchivaIndexManager {
return indexDirectory;
}
private Path getPackedIndexPath(Repository repo) throws IOException {
IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
Path repoDir = repo.getLocalPath();
URI indexDir = icf.getPackedIndexPath();
Path indexDirectory = null;
if ( ! StringUtils.isEmpty(indexDir.toString( ) ) )
{
indexDirectory = PathUtil.getPathFromUri( indexDir );
// not absolute so create it in repository directory
if ( !indexDirectory.isAbsolute( ) )
{
indexDirectory = repoDir.resolve( indexDirectory );
}
}
else
{
indexDirectory = repoDir.resolve( DEFAULT_PACKED_INDEX_DIR );
}
if ( !Files.exists( indexDirectory ) )
{
Files.createDirectories( indexDirectory );
}
return indexDirectory;
}
private IndexingContext createRemoteContext(RemoteRepository remoteRepository ) throws IOException
{
Path appServerBase = archivaConfiguration.getAppServerBaseDir( );

36
archiva-modules/archiva-base/archiva-proxy/src/test/java/org/apache/archiva/proxy/AbstractProxyTestCase.java
View File

@ -20,27 +20,19 @@ package org.apache.archiva.proxy;
*/
import net.sf.ehcache.CacheManager;
import org.apache.archiva.admin.model.beans.ManagedRepository;
import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
import org.apache.archiva.admin.repository.managed.DefaultManagedRepositoryAdmin;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.configuration.ManagedRepositoryConfiguration;
import org.apache.archiva.configuration.ProxyConnectorConfiguration;
import org.apache.archiva.configuration.RemoteRepositoryConfiguration;
import org.apache.archiva.policies.CachedFailuresPolicy;
import org.apache.archiva.policies.ChecksumPolicy;
import org.apache.archiva.policies.PropagateErrorsDownloadPolicy;
import org.apache.archiva.policies.PropagateErrorsOnUpdateDownloadPolicy;
import org.apache.archiva.policies.ReleasesPolicy;
import org.apache.archiva.policies.SnapshotsPolicy;
import org.apache.archiva.policies.*;
import org.apache.archiva.proxy.model.RepositoryProxyConnectors;
import org.apache.archiva.repository.ManagedRepositoryContent;
import org.apache.archiva.repository.RepositoryContentProvider;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.archiva.repository.maven2.MavenManagedRepository;
import org.apache.archiva.test.utils.ArchivaSpringJUnit4ClassRunner;
import org.apache.commons.io.FileUtils;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.wagon.Wagon;
@ -124,10 +116,6 @@ public abstract class AbstractProxyTestCase
WagonDelegate delegate;
@Inject
protected NexusIndexer nexusIndexer;
@Before
public void setUp()
throws Exception
@ -155,9 +143,6 @@ public abstract class AbstractProxyTestCase
applicationContext.getBean( RepositoryRegistry.class ).putRepository( repoConfig );
// to prevent windauze file leaking
removeMavenIndexes();
repositoryRegistry.setArchivaConfiguration( config );
// Setup target (proxied to) repository.
@ -199,25 +184,6 @@ public abstract class AbstractProxyTestCase
log.info( "\n.\\ {}() \\._________________________________________\n", name );
}
@After
public void shutdown()
throws Exception
{
removeMavenIndexes();
}
protected void removeMavenIndexes()
throws Exception
{
for ( IndexingContext indexingContext : nexusIndexer.getIndexingContexts().values() )
{
nexusIndexer.removeIndexingContext( indexingContext, false );
}
}
protected void assertChecksums( Path expectedFile, String expectedSha1Contents, String expectedMd5Contents )
throws Exception
{

29
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/AbstractRepository.java
View File

@ -54,6 +54,11 @@ public class AbstractRepository
private String indexDirectory;
/*
* @since 3.0.0
*/
private String packedIndexDirectory;
private String toStringCache = null;
@ -136,6 +141,7 @@ public class AbstractRepository
public void setIndexDirectory( String indexDirectory )
{
this.toStringCache=null;
this.indexDirectory = indexDirectory;
}
@ -202,9 +208,19 @@ public class AbstractRepository
}
public void setType(String type) {
toStringCache=null;
this.type = type;
}
public String getPackedIndexDirectory() {
return packedIndexDirectory;
}
public void setPackedIndexDirectory(String packedIndexDirectory) {
toStringCache=null;
this.packedIndexDirectory = packedIndexDirectory;
}
@Override
public String toString()
{
@ -214,12 +230,13 @@ public class AbstractRepository
{
final StringBuilder sb = new StringBuilder( );
sb.append( "AbstractRepository" );
sb.append( "{id='" ).append( id ).append( '\'' );
sb.append(", type='").append(type).append('\'');
sb.append( ", name='" ).append( getLocaleString( name ) ).append( '\'' );
sb.append( ", layout='" ).append( layout ).append( '\'' );
sb.append( ", indexDirectory='" ).append( indexDirectory ).append( '\'' );
sb.append( ", description='" ).append( getLocaleString( description ) ).append( '\'' );
sb.append( "{ id=\"" ).append( id ).append( '"' );
sb.append( ", type=\"").append(type).append('"');
sb.append( ", name=\"" ).append( getLocaleString( name ) ).append( '"' );
sb.append( ", layout=\"" ).append( layout ).append( '"' );
sb.append( ", indexDirectory=\"" ).append( indexDirectory ).append( '"' );
sb.append( ", packedIndexDirectory=\"").append(packedIndexDirectory).append('"');
sb.append( ", description=\"" ).append( getLocaleString( description ) ).append( '"' );
sb.append( '}' );
toStringCache=sb.toString( );
return toStringCache;

2
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/AbstractRepositoryAdmin.java
View File

@ -125,6 +125,7 @@ public abstract class AbstractRepositoryAdmin
if (repo.supportsFeature( IndexCreationFeature.class )) {
IndexCreationFeature icf = repo.getFeature( IndexCreationFeature.class ).get();
adminRepo.setIndexDirectory( convertUriToString( icf.getIndexPath() ) );
adminRepo.setPackedIndexDirectory(convertUriToString(icf.getPackedIndexPath()));
}
}
@ -134,6 +135,7 @@ public abstract class AbstractRepositoryAdmin
repoConfig.setLayout( repo.getLayout() );
repoConfig.setDescription( repo.getDescription() );
repoConfig.setIndexDir( repo.getIndexDirectory() );
repoConfig.setPackedIndexDir(repo.getPackedIndexDirectory());
repoConfig.setType( StringUtils.isEmpty( repo.getType() ) ? "MAVEN" : repo.getType() );
}

2
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/managed/DefaultManagedRepositoryAdmin.java
View File

@ -145,7 +145,6 @@ public class DefaultManagedRepositoryAdmin
adminRepo.setCronExpression( repo.getSchedulingDefinition() );
if (repo.supportsFeature( IndexCreationFeature.class )) {
IndexCreationFeature icf = repo.getFeature( IndexCreationFeature.class ).get();
adminRepo.setIndexDirectory(convertUriToString( icf.getIndexPath() ));
adminRepo.setSkipPackedIndexCreation( icf.isSkipPackedIndexCreation() );
}
adminRepo.setScanned( repo.isScanned() );
@ -180,6 +179,7 @@ public class DefaultManagedRepositoryAdmin
repoConfig.setDeleteReleasedSnapshots( repo.isDeleteReleasedSnapshots() );
repoConfig.setSkipPackedIndexCreation( repo.isSkipPackedIndexCreation());
repoConfig.setStageRepoNeeded( repo.isStageRepoNeeded() );
return repoConfig;
}

1
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/remote/DefaultRemoteRepositoryAdmin.java
View File

@ -216,6 +216,7 @@ public class DefaultRemoteRepositoryAdmin
Configuration configuration = getArchivaConfiguration().getConfiguration();
RemoteRepositoryConfiguration remoteRepositoryConfiguration =
getRepositoryConfiguration( remoteRepository );
log.debug("Adding remote repo {}", remoteRepositoryConfiguration);
try
{

9
archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/features/RemoteIndexFeature.java
View File

@ -133,4 +133,13 @@ public class RemoteIndexFeature implements RepositoryFeature<RemoteIndexFeature>
public boolean hasIndex() {
return this.indexUri!=null && !StringUtils.isEmpty( this.indexUri.getPath() );
}
@Override
public String toString() {
StringBuilder str = new StringBuilder();
return str.append("RemoteIndexFeature:{downloadRemoteIndex=").append(downloadRemoteIndex)
.append(",indexURI=").append(indexUri)
.append(",downloadOnStartup=").append(downloadRemoteIndexOnStartup)
.append(",timeout=").append(downloadTimeout).append("}").toString();
}
}

6
archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/AbstractRemoteRepository.java
View File

@ -151,4 +151,10 @@ public abstract class AbstractRemoteRepository extends AbstractRepository implem
return repositoryBase.resolve(getId());
}
@Override
public String toString() {
StringBuilder str = new StringBuilder();
return str.append("checkPath=").append(checkPath)
.append(",creds:").append(credentials).toString();
}
}

3
archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/RepositoryRegistry.java
View File

@ -21,7 +21,6 @@ package org.apache.archiva.repository;
import org.apache.archiva.configuration.*;
import org.apache.archiva.indexer.*;
import org.apache.archiva.metadata.model.facets.AuditEvent;
import org.apache.archiva.redback.components.registry.RegistryException;
import org.apache.archiva.repository.features.IndexCreationEvent;
import org.apache.archiva.repository.features.IndexCreationFeature;
@ -1017,7 +1016,7 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven
@Override
public <T> void raise(RepositoryEvent<T> event) {
if (event.getType().equals(IndexCreationEvent.Index.URI_CHANGE)) {
if (event instanceof IndexCreationEvent ) {
if (managedRepositories.containsKey(event.getRepository().getId()) ||
remoteRepositories.containsKey(event.getRepository().getId())) {
EditableRepository repo = (EditableRepository) event.getRepository();

11
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java
View File

@ -66,9 +66,6 @@ public class ArchivaIndexingTaskExecutor
@Inject
private ArtifactContextProducer artifactContextProducer;
@Inject
private ManagedRepositoryAdmin managedRepositoryAdmin;
@Inject
private NexusIndexer nexusIndexer;
@ -241,23 +238,25 @@ public class ArchivaIndexingTaskExecutor
try
{
log.debug("Finishing indexing");
context.optimize( );
if ( repository.supportsFeature( IndexCreationFeature.class ) )
{
IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class ).get( );
if ( !icf.isSkipPackedIndexCreation( ) )
if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath()!=null)
{
log.debug("Creating packed index from {} on {}", context.getIndexDirectoryFile(), icf.getLocalPackedIndexPath());
IndexPackingRequest request = new IndexPackingRequest( context, //
context.acquireIndexSearcher( ).getIndexReader( ),
//
context.getIndexDirectoryFile( ) );
icf.getLocalPackedIndexPath().toFile() );
indexPacker.packIndex( request );
context.updateTimestamp( true );
log.debug( "Index file packaged at '{}'.", context.getIndexDirectoryFile( ) );
log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath() );
} else {
log.debug( "skip packed index creation" );
}

63
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java
View File

@ -20,10 +20,7 @@ package org.apache.archiva.scheduler.indexing;
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.common.ArchivaException;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
import org.apache.archiva.configuration.ArchivaConfiguration;
@ -34,7 +31,6 @@ import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.archiva.repository.features.RemoteIndexFeature;
import org.apache.commons.lang.StringUtils;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
import org.apache.maven.index.packer.IndexPacker;
@ -46,7 +42,6 @@ import org.springframework.scheduling.support.CronTrigger;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.IOException;
@ -78,18 +73,9 @@ public class DefaultDownloadRemoteIndexScheduler
@Inject
private WagonFactory wagonFactory;
@Inject
private RemoteRepositoryAdmin remoteRepositoryAdmin;
@Inject
private ProxyConnectorAdmin proxyConnectorAdmin;
@Inject
private NetworkProxyAdmin networkProxyAdmin;
@Inject
private NexusIndexer nexusIndexer;
@Inject
private IndexUpdater indexUpdater;
@ -101,8 +87,8 @@ public class DefaultDownloadRemoteIndexScheduler
@PostConstruct
public void startup()
throws ArchivaException, RepositoryAdminException, PlexusSisuBridgeException, IOException,
UnsupportedExistingLuceneIndexException, DownloadRemoteIndexException, UnsupportedBaseContextException {
throws
DownloadRemoteIndexException, UnsupportedBaseContextException {
archivaConfiguration.addListener( this );
// TODO add indexContexts even if null
@ -129,22 +115,6 @@ public class DefaultDownloadRemoteIndexScheduler
}
@PreDestroy
public void shutdown()
throws RepositoryAdminException, IOException
{
for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() )
{
String contextKey = "remote-" + remoteRepository.getId();
IndexingContext context = nexusIndexer.getIndexingContexts().get( contextKey );
if ( context == null )
{
continue;
}
nexusIndexer.removeIndexingContext( context, false );
}
}
@Override
public void configurationEvent( ConfigurationEvent event )
{
@ -158,36 +128,41 @@ public class DefaultDownloadRemoteIndexScheduler
{
try
{
RemoteRepository remoteRepository = remoteRepositoryAdmin.getRemoteRepository( repositoryId );
if ( remoteRepository == null )
org.apache.archiva.repository.RemoteRepository remoteRepo = repositoryRegistry.getRemoteRepository(repositoryId);
if ( remoteRepo == null )
{
log.warn( "ignore scheduleDownloadRemote for repo with id {} as not exists", repositoryId );
return;
}
if (!remoteRepo.supportsFeature(RemoteIndexFeature.class)) {
log.warn("ignore scheduleDownloadRemote for repo with id {}. Does not support remote index.", repositoryId);
return;
}
RemoteIndexFeature rif = remoteRepo.getFeature(RemoteIndexFeature.class).get();
NetworkProxy networkProxy = null;
if ( StringUtils.isNotBlank( remoteRepository.getRemoteDownloadNetworkProxyId() ) )
if ( StringUtils.isNotBlank( rif.getProxyId() ) )
{
networkProxy = networkProxyAdmin.getNetworkProxy( remoteRepository.getRemoteDownloadNetworkProxyId() );
networkProxy = networkProxyAdmin.getNetworkProxy( rif.getProxyId() );
if ( networkProxy == null )
{
log.warn(
"your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
remoteRepository.getRemoteDownloadNetworkProxyId() );
rif.getProxyId() );
}
}
DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest = new DownloadRemoteIndexTaskRequest() //
.setRemoteRepository( remoteRepository ) //
.setRemoteRepository( remoteRepo ) //
.setNetworkProxy( networkProxy ) //
.setFullDownload( fullDownload ) //
.setWagonFactory( wagonFactory ) //
.setRemoteRepositoryAdmin( remoteRepositoryAdmin ) //
.setIndexUpdater( indexUpdater ) //
.setIndexPacker( this.indexPacker );
if ( now )
{
log.info( "schedule download remote index for repository {}", remoteRepository.getId() );
log.info( "schedule download remote index for repository {}", remoteRepo.getId() );
// do it now
taskScheduler.schedule(
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ),
@ -196,10 +171,10 @@ public class DefaultDownloadRemoteIndexScheduler
else
{
log.info( "schedule download remote index for repository {} with cron expression {}",
remoteRepository.getId(), remoteRepository.getCronExpression() );
remoteRepo.getId(), remoteRepo.getSchedulingDefinition());
try
{
CronTrigger cronTrigger = new CronTrigger( remoteRepository.getCronExpression() );
CronTrigger cronTrigger = new CronTrigger( remoteRepo.getSchedulingDefinition());
taskScheduler.schedule(
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ),
cronTrigger );
@ -209,11 +184,11 @@ public class DefaultDownloadRemoteIndexScheduler
log.warn( "Unable to schedule remote index download: {}", e.getLocalizedMessage() );
}
if ( remoteRepository.isDownloadRemoteIndexOnStartup() )
if ( rif.isDownloadRemoteIndexOnStartup() )
{
log.info(
"remote repository {} configured with downloadRemoteIndexOnStartup schedule now a download",
remoteRepository.getId() );
remoteRepo.getId() );
taskScheduler.schedule(
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ),
new Date() );

55
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java
View File

@ -19,15 +19,22 @@ package org.apache.archiva.scheduler.indexing;
*/
import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.archiva.proxy.common.WagonFactoryRequest;
import org.apache.archiva.repository.PasswordCredentials;
import org.apache.archiva.repository.RemoteRepository;
import org.apache.archiva.repository.RepositoryException;
import org.apache.archiva.repository.RepositoryType;
import org.apache.archiva.repository.features.RemoteIndexFeature;
import org.apache.commons.lang.time.StopWatch;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.updater.IndexUpdateRequest;
import org.apache.maven.index.updater.IndexUpdateResult;
import org.apache.maven.index.updater.IndexUpdater;
import org.apache.maven.index.updater.ResourceFetcher;
import org.apache.maven.index_shaded.lucene.index.IndexNotFoundException;
import org.apache.maven.wagon.ResourceDoesNotExistException;
import org.apache.maven.wagon.StreamWagon;
import org.apache.maven.wagon.TransferFailedException;
@ -65,8 +72,6 @@ public class DownloadRemoteIndexTask
private RemoteRepository remoteRepository;
private RemoteRepositoryAdmin remoteRepositoryAdmin;
private WagonFactory wagonFactory;
private NetworkProxy networkProxy;
@ -87,7 +92,6 @@ public class DownloadRemoteIndexTask
this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
this.runningRemoteDownloadIds = runningRemoteDownloadIds;
this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin();
}
@Override
@ -112,8 +116,17 @@ public class DownloadRemoteIndexTask
try
{
log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() );
IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository );
if (this.remoteRepository.getIndexingContext()==null) {
throw new IndexNotFoundException("No index context set for repository "+remoteRepository.getId());
}
if (this.remoteRepository.getType()!= RepositoryType.MAVEN) {
throw new RepositoryException("Bad repository type");
}
if (!this.remoteRepository.supportsFeature(RemoteIndexFeature.class)) {
throw new RepositoryException("Repository does not support RemotIndexFeature "+remoteRepository.getId());
}
RemoteIndexFeature rif = this.remoteRepository.getFeature(RemoteIndexFeature.class).get();
IndexingContext indexingContext = this.remoteRepository.getIndexingContext().getBaseContext(IndexingContext.class);
// create a temp directory to download files
tempIndexDirectory = Paths.get(indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" );
@ -126,22 +139,22 @@ public class DownloadRemoteIndexTask
tempIndexDirectory.toFile().deleteOnExit();
String baseIndexUrl = indexingContext.getIndexUpdateUrl();
String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol();
String wagonProtocol = this.remoteRepository.getLocation().getScheme();
final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon(
new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy(
this.networkProxy )
);
// FIXME olamy having 2 config values
wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
wagon.setTimeout( remoteRepository.getTimeout() * 1000 );
wagon.setReadTimeout( (int)rif.getDownloadTimeout().toMillis());
wagon.setTimeout( (int)remoteRepository.getTimeout().toMillis());
if ( wagon instanceof AbstractHttpClientWagon )
{
HttpConfiguration httpConfiguration = new HttpConfiguration();
HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration();
httpMethodConfiguration.setUsePreemptive( true );
httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 );
httpMethodConfiguration.setReadTimeout( (int)rif.getDownloadTimeout().toMillis() );
httpConfiguration.setGet( httpMethodConfiguration );
AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration );
}
@ -158,12 +171,14 @@ public class DownloadRemoteIndexTask
proxyInfo.setPassword( this.networkProxy.getPassword() );
}
AuthenticationInfo authenticationInfo = null;
if ( this.remoteRepository.getUserName() != null )
if ( this.remoteRepository.getLoginCredentials()!=null && this.remoteRepository.getLoginCredentials() instanceof PasswordCredentials )
{
PasswordCredentials creds = (PasswordCredentials) this.remoteRepository.getLoginCredentials();
authenticationInfo = new AuthenticationInfo();
authenticationInfo.setUserName( this.remoteRepository.getUserName() );
authenticationInfo.setPassword( this.remoteRepository.getPassword() );
authenticationInfo.setUserName( creds.getUsername());
authenticationInfo.setPassword( new String(creds.getPassword()) );
}
log.debug("Connection to {}, authInfo={}", this.remoteRepository.getId(), authenticationInfo);
wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
proxyInfo );
@ -172,6 +187,8 @@ public class DownloadRemoteIndexTask
{
Files.createDirectories( indexDirectory );
}
log.debug("Downloading index file to {}", indexDirectory);
log.debug("Index cache dir {}", indexCacheDirectory);
ResourceFetcher resourceFetcher =
new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository );
@ -179,10 +196,11 @@ public class DownloadRemoteIndexTask
request.setForceFullUpdate( this.fullDownload );
request.setLocalIndexCacheDir( indexCacheDirectory.toFile() );
this.indexUpdater.fetchAndUpdateIndex( request );
IndexUpdateResult result = this.indexUpdater.fetchAndUpdateIndex(request);
log.debug("Update result success: {}", result.isSuccessful());
stopWatch.stop();
log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(),
( stopWatch.getTime() / 1000 ) );
log.info( "time update index from remote for repository {}: {}ms", this.remoteRepository.getId(),
( stopWatch.getTime() ) );
// index packing optionnal ??
//IndexPackingRequest indexPackingRequest =
@ -241,6 +259,7 @@ public class DownloadRemoteIndexTask
{
this.totalLength = 0;
resourceName = transferEvent.getResource().getName();
log.info("Transferring: {}, {}", transferEvent.getResource().getContentLength(), transferEvent.getLocalFile().toString());
log.info( "start transfer of {}", transferEvent.getResource().getName() );
}
@ -256,8 +275,8 @@ public class DownloadRemoteIndexTask
{
resourceName = transferEvent.getResource().getName();
long endTime = System.currentTimeMillis();
log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(),
this.totalLength / 1024, ( endTime - startTime ) / 1000 );
log.info( "end of transfer file {}: {}b, {}ms", transferEvent.getResource().getName(),
this.totalLength, ( endTime - startTime ) );
}
@Override

15
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java
View File

@ -19,9 +19,9 @@ package org.apache.archiva.scheduler.indexing;
*/
import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.archiva.repository.RemoteRepository;
import org.apache.maven.index.packer.IndexPacker;
import org.apache.maven.index.updater.IndexUpdater;
@ -33,8 +33,6 @@ public class DownloadRemoteIndexTaskRequest
{
private RemoteRepository remoteRepository;
private RemoteRepositoryAdmin remoteRepositoryAdmin;
private WagonFactory wagonFactory;
private NetworkProxy networkProxy;
@ -106,17 +104,6 @@ public class DownloadRemoteIndexTaskRequest
return this;
}
public RemoteRepositoryAdmin getRemoteRepositoryAdmin()
{
return remoteRepositoryAdmin;
}
public DownloadRemoteIndexTaskRequest setRemoteRepositoryAdmin( RemoteRepositoryAdmin remoteRepositoryAdmin )
{
this.remoteRepositoryAdmin = remoteRepositoryAdmin;
return this;
}
public IndexPacker getIndexPacker()
{
return indexPacker;

22
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutorTest.java
View File

@ -160,7 +160,7 @@ public class ArchivaIndexingTaskExecutorTest
FlatSearchResponse response = indexer.searchFlat( request );
assertTrue( Files.exists(basePath.resolve( ".indexer" )) );
assertFalse( Files.exists(basePath.resolve(".index" )) );
assertTrue( Files.exists(basePath.resolve(".index" )) );
assertEquals( 1, response.getTotalHits() );
Set<ArtifactInfo> results = response.getResults();
@ -204,7 +204,7 @@ public class ArchivaIndexingTaskExecutorTest
ctx.releaseIndexSearcher( searcher );
assertTrue( Files.exists(basePath.resolve(".indexer" )) );
assertFalse( Files.exists(basePath.resolve(".index" )) );
assertTrue( Files.exists(basePath.resolve(".index" )) );
// should only return 1 hit!
assertEquals( 1, topDocs.totalHits );
@ -239,7 +239,7 @@ public class ArchivaIndexingTaskExecutorTest
FlatSearchResponse response = indexer.searchFlat( flatSearchRequest );
assertTrue( Files.exists(basePath.resolve(".indexer" )) );
assertFalse( Files.exists(basePath.resolve( ".index" )) );
assertTrue( Files.exists(basePath.resolve( ".index" )) );
// should return 1 hit
assertEquals( 1, response.getTotalHitsCount() );
@ -261,7 +261,7 @@ public class ArchivaIndexingTaskExecutorTest
BooleanClause.Occur.SHOULD );
assertTrue( Files.exists(basePath.resolve( ".indexer" )) );
assertFalse( Files.exists(basePath.resolve(".index" )) );
assertTrue( Files.exists(basePath.resolve(".index" )) );
flatSearchRequest = new FlatSearchRequest( q, getIndexingContext() );
@ -278,9 +278,9 @@ public class ArchivaIndexingTaskExecutorTest
{
Path basePath = PathUtil.getPathFromUri( repositoryConfig.getLocation());
Path indexerDirectory =basePath.resolve( ".indexer" );
Path indexDirectory = basePath.resolve(".index");
Files.list(indexerDirectory).filter( path -> path.getFileName().toString().startsWith("nexus-maven-repository-index") )
Files.list(indexDirectory).filter( path -> path.getFileName().toString().startsWith("nexus-maven-repository-index") )
.forEach( path ->
{
try
@ -311,19 +311,19 @@ public class ArchivaIndexingTaskExecutorTest
indexingExecutor.executeTask( task );
assertTrue( Files.exists(indexerDirectory) );
assertTrue( Files.exists(indexDirectory) );
// test packed index file creation
//no more zip
//Assertions.assertThat(new File( indexerDirectory, "nexus-maven-repository-index.zip" )).exists();
Assertions.assertThat( Files.exists(indexerDirectory.resolve("nexus-maven-repository-index.properties" ) ));
Assertions.assertThat( Files.exists(indexerDirectory.resolve("nexus-maven-repository-index.gz" ) ));
Assertions.assertThat( Files.exists(indexDirectory.resolve("nexus-maven-repository-index.properties" ) ));
Assertions.assertThat( Files.exists(indexDirectory.resolve("nexus-maven-repository-index.gz" ) ));
// unpack .zip index
Path destDir = basePath.resolve( ".indexer/tmp" );
Path destDir = basePath.resolve( ".index/tmp" );
//unzipIndex( indexerDirectory.getPath(), destDir.getPath() );
DefaultIndexUpdater.FileFetcher fetcher = new DefaultIndexUpdater.FileFetcher( indexerDirectory.toFile() );
DefaultIndexUpdater.FileFetcher fetcher = new DefaultIndexUpdater.FileFetcher( indexDirectory.toFile() );
IndexUpdateRequest updateRequest = new IndexUpdateRequest( getIndexingContext(), fetcher );
//updateRequest.setLocalIndexCacheDir( indexerDirectory );
indexUpdater.fetchAndUpdateIndex( updateRequest );

5
archiva-modules/archiva-web/archiva-rest/archiva-rest-services/src/test/java/org/apache/archiva/rest/services/AbstractArchivaRestTest.java
View File

@ -447,8 +447,11 @@ public abstract class AbstractArchivaRestTest
}
managedRepository.setLocation( Paths.get( repoPath ).toString() );
String suffix = Long.toString( new Date().getTime() );
String baseDir = System.getProperty("java.io.tmpdir");
managedRepository.setIndexDirectory(
System.getProperty( "java.io.tmpdir" ) + "/.index-" + Long.toString( new Date().getTime() ) );
baseDir + "/.indexer-" + suffix );
managedRepository.setPackedIndexDirectory(baseDir + "/.index-" + suffix);
managedRepository.setStageRepoNeeded( stageNeeded );
managedRepository.setSnapshots( true );

2
archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexTest.java
View File

@ -161,7 +161,7 @@ public class DownloadMergedIndexTest
remoteRepository.setName( "all-merged" );
remoteRepository.setDownloadRemoteIndex( true );
remoteRepository.setUrl( "http://localhost:" + port + "/repository/test-group" );
remoteRepository.setRemoteIndexUrl( "http://localhost:" + port + "/repository/test-group/.indexer" );
remoteRepository.setRemoteIndexUrl( "http://localhost:" + port + "/repository/test-group/.index" );
remoteRepository.setUserName( RedbackRoleConstants.ADMINISTRATOR_ACCOUNT_NAME );
remoteRepository.setPassword( FakeCreateAdminService.ADMIN_TEST_PWD );

5
archiva-modules/plugins/maven2-repository/src/main/java/org/apache/archiva/repository/maven2/MavenRemoteRepository.java
View File

@ -108,5 +108,8 @@ public class MavenRemoteRepository extends AbstractRemoteRepository
}
}
@Override
public String toString() {
return super.toString()+", remoteIndexFeature="+remoteIndexFeature.toString()+", indexCreationFeature="+indexCreationFeature.toString();
}
}

33
archiva-modules/plugins/maven2-repository/src/main/java/org/apache/archiva/repository/maven2/MavenRepositoryProvider.java
View File

@ -150,6 +150,9 @@ public class MavenRepositoryProvider implements RepositoryProvider {
IndexCreationFeature indexCreationFeature = repo.getFeature(IndexCreationFeature.class).get();
indexCreationFeature.setSkipPackedIndexCreation(cfg.isSkipPackedIndexCreation());
indexCreationFeature.setIndexPath(getURIFromString(cfg.getIndexDir()));
indexCreationFeature.setPackedIndexPath(getURIFromString(cfg.getPackedIndexDir()));
/* -> Should be created by MavenIndexProvider
Path indexPath;
if (indexCreationFeature.getIndexPath().getScheme() == null) {
indexPath = Paths.get(indexCreationFeature.getIndexPath().getPath());
@ -167,7 +170,7 @@ public class MavenRepositoryProvider implements RepositoryProvider {
} catch (IOException e) {
log.error("Could not create index directory {}", absoluteIndexPath);
throw new RepositoryException("Could not create index directory " + absoluteIndexPath);
}
}*/
ArtifactCleanupFeature artifactCleanupFeature = repo.getFeature(ArtifactCleanupFeature.class).get();
@ -237,10 +240,14 @@ public class MavenRepositoryProvider implements RepositoryProvider {
} else {
credentials.setPassword(new char[0]);
}
IndexCreationFeature indexCreationFeature = repo.getFeature(IndexCreationFeature.class).get();
if (cfg.getIndexDir() != null) {
IndexCreationFeature indexCreationFeature = repo.getFeature(IndexCreationFeature.class).get();
indexCreationFeature.setIndexPath(getURIFromString(cfg.getIndexDir()));
}
if (cfg.getPackedIndexDir() != null) {
indexCreationFeature.setPackedIndexPath(getURIFromString(cfg.getPackedIndexDir()));
}
log.debug("Updated remote instance {}", repo);
}
@Override
@ -272,6 +279,7 @@ public class MavenRepositoryProvider implements RepositoryProvider {
IndexCreationFeature indexCreationFeature = remoteRepository.getFeature(IndexCreationFeature.class).get();
cfg.setIndexDir(convertUriToPath(indexCreationFeature.getIndexPath()));
cfg.setPackedIndexDir(convertUriToPath(indexCreationFeature.getPackedIndexPath()));
RemoteIndexFeature remoteIndexFeature = remoteRepository.getFeature(RemoteIndexFeature.class).get();
cfg.setRemoteIndexUrl(remoteIndexFeature.getIndexUri().toString());
@ -305,6 +313,7 @@ public class MavenRepositoryProvider implements RepositoryProvider {
cfg.setStageRepoNeeded(stagingRepositoryFeature.isStageRepoNeeded());
IndexCreationFeature indexCreationFeature = managedRepository.getFeature(IndexCreationFeature.class).get();
cfg.setIndexDir(convertUriToPath(indexCreationFeature.getIndexPath()));
cfg.setPackedIndexDir(convertUriToPath(indexCreationFeature.getPackedIndexPath()));
cfg.setSkipPackedIndexCreation(indexCreationFeature.isSkipPackedIndexCreation());
ArtifactCleanupFeature artifactCleanupFeature = managedRepository.getFeature(ArtifactCleanupFeature.class).get();
@ -345,7 +354,7 @@ public class MavenRepositoryProvider implements RepositoryProvider {
try {
indexDir = Paths.get(new URI(repository.getIndexDir().startsWith("file://") ? repository.getIndexDir() : "file://" + repository.getIndexDir()));
if (indexDir.isAbsolute()) {
Path newDir = Paths.get(new URI(stagingRepository.getLocation().startsWith("file://") ? stagingRepository.getLocation() : "file://" + stagingRepository.getLocation())).resolve(".index");
Path newDir = indexDir.getParent().resolve(indexDir.getFileName() + StagingRepositoryFeature.STAGING_REPO_POSTFIX);
log.debug("Changing index directory {} -> {}", indexDir, newDir);
stagingRepository.setIndexDir(newDir.toString());
} else {
@ -358,6 +367,24 @@ public class MavenRepositoryProvider implements RepositoryProvider {
}
// in case of absolute dir do not use the same
}
if (StringUtils.isNotBlank(repository.getPackedIndexDir())) {
Path packedIndexDir = null;
try {
packedIndexDir = Paths.get(new URI(repository.getPackedIndexDir().startsWith("file://") ? repository.getPackedIndexDir() : "file://" + repository.getPackedIndexDir()));
if (packedIndexDir.isAbsolute()) {
Path newDir = packedIndexDir.getParent().resolve(packedIndexDir.getFileName() + StagingRepositoryFeature.STAGING_REPO_POSTFIX);
log.debug("Changing index directory {} -> {}", packedIndexDir, newDir);
stagingRepository.setPackedIndexDir(newDir.toString());
} else {
log.debug("Keeping index directory {}", repository.getPackedIndexDir());
stagingRepository.setPackedIndexDir(repository.getPackedIndexDir());
}
} catch (URISyntaxException e) {
log.error("Could not parse index path as uri {}", repository.getPackedIndexDir());
stagingRepository.setPackedIndexDir("");
}
// in case of absolute dir do not use the same
}
stagingRepository.setRefreshCronExpression(repository.getRefreshCronExpression());
stagingRepository.setReleases(repository.isReleases());
stagingRepository.setRetentionCount(repository.getRetentionCount());