Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[MRM-1524] downloading (optionnaly) remote index to display remote artifacts in search results 

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1175928 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Olivier Lamy 2011-09-26 16:07:16 +00:00
parent b42ae3b997
commit d1a54e99e8
33 changed files with 1639 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
archiva-modules/archiva-base/archiva-configuration/pom.xml
View File

@ -138,7 +138,7 @@
</execution>
</executions>
<configuration>
<version>1.3.0</version>
<version>1.4.0</version>
<models>
<model>src/main/mdo/configuration.mdo</model>
</models>

55
archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo
View File

@ -403,6 +403,24 @@
<!-- TODO: should be able to detect this from the repository (perhaps by metadata at the root) -->
<defaultValue>default</defaultValue>
</field>
<field>
<name>refreshCronExpression</name>
<version>1.0.0+</version>
<type>String</type>
<description>
When to run the refresh task.
Default is every hour
</description>
<defaultValue>0 0 * * * ?</defaultValue>
</field>
<field>
<name>indexDir</name>
<version>1.0.0+</version>
<type>String</type>
<description>
The directory for the indexes of this repository.
</description>
</field>
</fields>
<codeSegments>
<codeSegment>
@ -474,7 +492,24 @@
Timeout in seconds for connections to this repository
</description>
<defaultValue>60</defaultValue>
</field>
</field>
<field>
<name>downloadRemoteIndex</name>
<version>1.4.0+</version>
<type>boolean</type>
<description>
Activate download of remote index if remoteIndexUrl is set too.
</description>
<defaultValue>false</defaultValue>
</field>
<field>
<name>remoteIndexUrl</name>
<version>1.4.0+</version>
<type>String</type>
<description>
Remote Index Url : if not starting with http will be relative to the remote repository url.
</description>
</field>
</fields>
</class>
<class>
@ -519,24 +554,6 @@
<description>True if this repository should be scanned and processed.</description>
<defaultValue>true</defaultValue>
</field>
<field>
<name>indexDir</name>
<version>1.0.0+</version>
<type>String</type>
<description>
The directory for the indexes of this repository.
</description>
</field>
<field>
<name>refreshCronExpression</name>
<version>1.0.0+</version>
<type>String</type>
<description>
When to run the refresh task.
Default is every hour
</description>
<defaultValue>0 0 * * * ?</defaultValue>
</field>
<field>
<name>retentionCount</name>
<version>1.0.0+</version>

30
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/AbstractRepository.java → archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/AbstractRepository.java
View File

@ -1,4 +1,4 @@
package org.apache.archiva.admin.model;
package org.apache.archiva.admin.model.beans;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -34,6 +34,13 @@ public class AbstractRepository
private String layout = "default";
/**
* default model value
*/
private String cronExpression = "0 0 * * * ?";
private String indexDirectory;
public AbstractRepository()
{
// no op
@ -76,6 +83,25 @@ public class AbstractRepository
this.layout = layout;
}
public String getCronExpression()
{
return cronExpression;
}
public void setCronExpression( String cronExpression )
{
this.cronExpression = cronExpression;
}
public String getIndexDirectory()
{
return indexDirectory;
}
public void setIndexDirectory( String indexDirectory )
{
this.indexDirectory = indexDirectory;
}
public int hashCode()
{
@ -110,6 +136,8 @@ public class AbstractRepository
sb.append( "{id='" ).append( id ).append( '\'' );
sb.append( ", name='" ).append( name ).append( '\'' );
sb.append( ", layout='" ).append( layout ).append( '\'' );
sb.append( ", cronExpression='" ).append( cronExpression ).append( '\'' );
sb.append( ", indexDirectory='" ).append( indexDirectory ).append( '\'' );
sb.append( '}' );
return sb.toString();
}

32
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/ManagedRepository.java
View File

@ -19,8 +19,6 @@ package org.apache.archiva.admin.model.beans;
* under the License.
*/
import org.apache.archiva.admin.model.AbstractRepository;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.Serializable;
@ -42,11 +40,6 @@ public class ManagedRepository
private boolean blockRedeployments = false;
/**
* default model value
*/
private String cronExpression = "0 0 * * * ?";
/**
* not need when creating the repo : only available when reading
@ -55,7 +48,6 @@ public class ManagedRepository
private boolean scanned = false;
private String indexDirectory;
/**
* default model value
@ -89,8 +81,8 @@ public class ManagedRepository
this.snapshots = snapshots;
this.releases = releases;
this.blockRedeployments = blockRedeployments;
this.cronExpression = cronExpression;
this.indexDirectory = indexDir;
this.setCronExpression( cronExpression );
this.setIndexDirectory( indexDir );
this.scanned = scanned;
this.daysOlder = daysOlder;
this.retentionCount = retentionCount;
@ -144,15 +136,6 @@ public class ManagedRepository
this.blockRedeployments = blockRedeployments;
}
public String getCronExpression()
{
return cronExpression;
}
public void setCronExpression( String cronExpression )
{
this.cronExpression = cronExpression;
}
public ManagedRepository getStagingRepository()
{
@ -175,15 +158,7 @@ public class ManagedRepository
this.scanned = scanned;
}
public String getIndexDirectory()
{
return indexDirectory;
}
public void setIndexDirectory( String indexDirectory )
{
this.indexDirectory = indexDirectory;
}
public int getDaysOlder()
{
@ -239,15 +214,14 @@ public class ManagedRepository
public String toString()
{
final StringBuilder sb = new StringBuilder();
sb.append( super.toString() );
sb.append( "ManagedRepository" );
sb.append( "{location='" ).append( location ).append( '\'' );
sb.append( ", snapshots=" ).append( snapshots );
sb.append( ", releases=" ).append( releases );
sb.append( ", blockRedeployments=" ).append( blockRedeployments );
sb.append( ", cronExpression='" ).append( cronExpression ).append( '\'' );
sb.append( ", stagingRepository=" ).append( stagingRepository );
sb.append( ", scanned=" ).append( scanned );
sb.append( ", indexDirectory='" ).append( indexDirectory ).append( '\'' );
sb.append( ", daysOlder=" ).append( daysOlder );
sb.append( ", retentionCount=" ).append( retentionCount );
sb.append( ", deleteReleasedSnapshots=" ).append( deleteReleasedSnapshots );

38
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RemoteRepository.java
View File

@ -19,8 +19,6 @@ package org.apache.archiva.admin.model.beans;
* under the License.
*/
import org.apache.archiva.admin.model.AbstractRepository;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.Serializable;
@ -42,6 +40,17 @@ public class RemoteRepository
private int timeout = 60;
/**
* Activate download of remote index if remoteIndexUrl is set too.
*/
private boolean downloadRemoteIndex = false;
/**
* Remote Index Url : if not starting with http will be relative to the remote repository url.
*/
private String remoteIndexUrl = ".index";
public RemoteRepository()
{
// no op
@ -103,17 +112,40 @@ public class RemoteRepository
this.timeout = timeout;
}
public boolean isDownloadRemoteIndex()
{
return downloadRemoteIndex;
}
public void setDownloadRemoteIndex( boolean downloadRemoteIndex )
{
this.downloadRemoteIndex = downloadRemoteIndex;
}
public String getRemoteIndexUrl()
{
return remoteIndexUrl;
}
public void setRemoteIndexUrl( String remoteIndexUrl )
{
this.remoteIndexUrl = remoteIndexUrl;
}
@Override
public String toString()
{
final StringBuilder sb = new StringBuilder();
sb.append( super.toString() );
sb.append( "RemoteRepository" );
sb.append( "{url='" ).append( url ).append( '\'' );
sb.append( ", userName='" ).append( userName ).append( '\'' );
sb.append( ", password='" ).append( password ).append( '\'' );
sb.append( ", timeout=" ).append( timeout );
sb.append( ", downloadRemoteIndex=" ).append( downloadRemoteIndex );
sb.append( ", remoteIndexUrl='" ).append( remoteIndexUrl ).append( '\'' );
sb.append( '}' );
sb.append( super.toString() );
return sb.toString();
}

2
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/RepositoryCommonValidator.java
View File

@ -18,7 +18,7 @@ package org.apache.archiva.admin.repository;
* under the License.
*/
import org.apache.archiva.admin.model.AbstractRepository;
import org.apache.archiva.admin.model.beans.AbstractRepository;
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.validator.GenericValidator;

15
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/remote/DefaultRemoteRepositoryAdmin.java
View File

@ -24,10 +24,10 @@ import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.admin.repository.AbstractRepositoryAdmin;
import org.apache.archiva.audit.AuditEvent;
import org.apache.commons.lang.StringUtils;
import org.apache.archiva.configuration.Configuration;
import org.apache.archiva.configuration.ProxyConnectorConfiguration;
import org.apache.archiva.configuration.RemoteRepositoryConfiguration;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@ -52,11 +52,16 @@ public class DefaultRemoteRepositoryAdmin
List<RemoteRepository> remoteRepositories = new ArrayList<RemoteRepository>();
for ( RemoteRepositoryConfiguration repositoryConfiguration : getArchivaConfiguration().getConfiguration().getRemoteRepositories() )
{
remoteRepositories.add(
RemoteRepository remoteRepository =
new RemoteRepository( repositoryConfiguration.getId(), repositoryConfiguration.getName(),
repositoryConfiguration.getUrl(), repositoryConfiguration.getLayout(),
repositoryConfiguration.getUsername(), repositoryConfiguration.getPassword(),
repositoryConfiguration.getTimeout() ) );
repositoryConfiguration.getTimeout() );
remoteRepository.setDownloadRemoteIndex( repositoryConfiguration.isDownloadRemoteIndex() );
remoteRepository.setRemoteIndexUrl( repositoryConfiguration.getRemoteIndexUrl() );
remoteRepository.setCronExpression( repositoryConfiguration.getRefreshCronExpression() );
remoteRepository.setIndexDirectory( repositoryConfiguration.getIndexDir() );
remoteRepositories.add( remoteRepository );
}
return remoteRepositories;
}
@ -186,6 +191,10 @@ public class DefaultRemoteRepositoryAdmin
remoteRepositoryConfiguration.setUsername( remoteRepository.getUserName() );
remoteRepositoryConfiguration.setLayout( remoteRepository.getLayout() );
remoteRepositoryConfiguration.setName( remoteRepository.getName() );
remoteRepositoryConfiguration.setDownloadRemoteIndex( remoteRepository.isDownloadRemoteIndex() );
remoteRepositoryConfiguration.setRemoteIndexUrl( remoteRepository.getRemoteIndexUrl() );
remoteRepositoryConfiguration.setRefreshCronExpression( remoteRepository.getCronExpression() );
remoteRepositoryConfiguration.setIndexDir( remoteRepository.getIndexDirectory() );
return remoteRepositoryConfiguration;
}

2
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/utils/RepositoryComparator.java
View File

@ -18,7 +18,7 @@ package org.apache.archiva.admin.repository.utils;
* under the License.
*/
import org.apache.archiva.admin.model.AbstractRepository;
import org.apache.archiva.admin.model.beans.AbstractRepository;
import java.util.Comparator;

13
archiva-modules/archiva-base/archiva-repository-scanner/src/test/java/org/apache/archiva/repository/scanner/ArchivaAdministrationStub.java
View File

@ -24,6 +24,7 @@ import org.apache.archiva.admin.model.admin.ArchivaAdministration;
import org.apache.archiva.admin.model.beans.FileType;
import org.apache.archiva.admin.model.beans.LegacyArtifactPath;
import org.apache.archiva.admin.model.beans.OrganisationInformation;
import org.apache.archiva.admin.model.beans.UiConfiguration;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.springframework.stereotype.Service;
@ -163,4 +164,16 @@ public class ArchivaAdministrationStub
{
}
public UiConfiguration getUiConfiguration()
throws RepositoryAdminException
{
return null;
}
public void updateUiConfiguration( UiConfiguration uiConfiguration )
throws RepositoryAdminException
{
//To change body of implemented methods use File | Settings | File Templates.
}
}

81
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/pom.xml
View File

@ -27,6 +27,11 @@
</parent>
<artifactId>archiva-scheduler-indexing</artifactId>
<name>Archiva Scheduler :: Indexing</name>
<properties>
<jettyVersion>7.4.5.v20110725</jettyVersion>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.archiva</groupId>
@ -48,6 +53,10 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
@ -56,15 +65,85 @@
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-plexus-bridge</artifactId>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-proxy-common</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jettyVersion}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId>
<version>${jettyVersion}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<artifactId>slf4j-log4j12</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-repository-admin-default</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>metadata-store-file</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.codehaus.redback</groupId>
<artifactId>redback-rbac-cached</artifactId>
<scope>test</scope>
<version>${redback.version}</version>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<plexus.home>${project.build.outputDirectory}</plexus.home>
<appserver.base>${basedir}/target/appserver-base</appserver.base>
<java.io.tmpdir>${project.build.outputDirectory}</java.io.tmpdir>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>

18
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java
View File

@ -255,24 +255,6 @@ public class ArchivaIndexingTaskExecutor
throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'",
e );
}
finally
{
/*
olamy don't close it anymore as it nullify IndexSearcher
if ( context != null )
{
try
{
context.close( false );
}
catch ( IOException e )
{
log.error( "Error occurred while closing context: " + e.getMessage() );
throw new TaskExecutionException( "Error occurred while closing context: " + e.getMessage() );
}
}
*/
}
}
public void setIndexPacker( IndexPacker indexPacker )

245
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java Normal file
View File

@ -0,0 +1,245 @@
package org.apache.archiva.scheduler.indexing;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.ProxyConnector;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin;
import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.common.ArchivaException;
import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.configuration.ConfigurationEvent;
import org.apache.archiva.configuration.ConfigurationListener;
import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.commons.lang.StringUtils;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
import org.apache.maven.index.updater.IndexUpdater;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.support.CronTrigger;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
/**
* @author Olivier Lamy
* @since 1.4
*/
@Service( "downloadRemoteIndexScheduler#default" )
public class DefaultDownloadRemoteIndexScheduler
implements ConfigurationListener, DownloadRemoteIndexScheduler
{
private Logger log = LoggerFactory.getLogger( getClass() );
@Inject
@Named( value = "taskScheduler#indexDownloadRemote" )
private TaskScheduler taskScheduler;
@Inject
private ArchivaConfiguration archivaConfiguration;
@Inject
private WagonFactory wagonFactory;
@Inject
private RemoteRepositoryAdmin remoteRepositoryAdmin;
@Inject
private ProxyConnectorAdmin proxyConnectorAdmin;
@Inject
private NetworkProxyAdmin networkProxyAdmin;
@Inject
private PlexusSisuBridge plexusSisuBridge;
@Inject
private MavenIndexerUtils mavenIndexerUtils;
private NexusIndexer nexusIndexer;
private IndexUpdater indexUpdater;
// store ids about currently running remote download : updated in DownloadRemoteIndexTask
private List<String> runningRemoteDownloadIds = new CopyOnWriteArrayList<String>();
@PostConstruct
public void startup()
throws ArchivaException, RepositoryAdminException, PlexusSisuBridgeException, IOException,
UnsupportedExistingLuceneIndexException, DownloadRemoteIndexException
{
archivaConfiguration.addListener( this );
// TODO add indexContexts even if null
// FIXME get this from ArchivaAdministration
String appServerBase = System.getProperty( "appserver.base" );
nexusIndexer = plexusSisuBridge.lookup( NexusIndexer.class );
indexUpdater = plexusSisuBridge.lookup( IndexUpdater.class );
for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() )
{
String contextKey = "remote-" + remoteRepository.getId();
if ( nexusIndexer.getIndexingContexts().get( contextKey ) != null )
{
continue;
}
// create path
File repoDir = new File( appServerBase, "data/remotes/" + remoteRepository.getId() );
if ( !repoDir.exists() )
{
repoDir.mkdirs();
}
File indexDirectory = new File( repoDir, ".index" );
if ( !indexDirectory.exists() )
{
indexDirectory.mkdirs();
}
nexusIndexer.addIndexingContext( contextKey, remoteRepository.getId(), repoDir, indexDirectory,
remoteRepository.getUrl(), calculateIndexRemoteUrl( remoteRepository ),
mavenIndexerUtils.getAllIndexCreators() );
// TODO record jobs from configuration
if ( remoteRepository.isDownloadRemoteIndex() && StringUtils.isNotEmpty(
remoteRepository.getCronExpression() ) )
{
boolean fullDownload = indexDirectory.list().length == 0;
scheduleDownloadRemote( remoteRepository.getId(), false, fullDownload );
}
}
}
public void configurationEvent( ConfigurationEvent event )
{
// TODO remove jobs and add again
}
public void scheduleDownloadRemote( String repositoryId, boolean now, boolean fullDownload )
throws DownloadRemoteIndexException
{
try
{
RemoteRepository remoteRepository = remoteRepositoryAdmin.getRemoteRepository( repositoryId );
if ( remoteRepository == null )
{
log.warn( "ignore scheduleDownloadRemote for repo with id {} as not exists", repositoryId );
return;
}
String networkProxyId = null;
for ( ProxyConnector proxyConnector : proxyConnectorAdmin.getProxyConnectors() )
{
if ( StringUtils.equals( proxyConnector.getTargetRepoId(), repositoryId ) )
{
networkProxyId = proxyConnector.getProxyId();
break;
}
}
// FIXME add a field networkProxy at the remoteRepositories level : only use for remote index download
NetworkProxy networkProxy = null;
if ( networkProxyId != null )
{
for ( NetworkProxy np : networkProxyAdmin.getNetworkProxies() )
{
if ( StringUtils.equals( np.getId(), networkProxyId ) )
{
networkProxy = np;
break;
}
}
}
//archivaConfiguration.getConfiguration().getProxyConnectorAsMap().get( "" ).get( 0 ).
//archivaConfiguration.getConfiguration().getNetworkProxiesAsMap()
DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest =
new DownloadRemoteIndexTaskRequest().setRemoteRepository( remoteRepository ).setNetworkProxy(
networkProxy ).setFullDownload( fullDownload ).setWagonFactory( wagonFactory ).setNexusIndexer(
nexusIndexer ).setIndexUpdater( indexUpdater );
if ( now )
{
// do it in async
taskScheduler.schedule(
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ),
new Date() );
}
else
{
taskScheduler.schedule(
new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ),
new CronTrigger( remoteRepository.getCronExpression() ) );
}
}
catch ( RepositoryAdminException e )
{
log.error( e.getMessage(), e );
throw new DownloadRemoteIndexException( e.getMessage(), e );
}
}
protected String calculateIndexRemoteUrl( RemoteRepository remoteRepository )
{
if ( StringUtils.startsWith( remoteRepository.getRemoteIndexUrl(), "http" ) )
{
String baseUrl = remoteRepository.getRemoteIndexUrl();
return baseUrl.endsWith( "/" ) ? StringUtils.substringBeforeLast( baseUrl, "/" ) : baseUrl;
}
String baseUrl = StringUtils.endsWith( remoteRepository.getUrl(), "/" ) ? StringUtils.substringBeforeLast(
remoteRepository.getUrl(), "/" ) : remoteRepository.getUrl();
baseUrl = StringUtils.isEmpty( remoteRepository.getRemoteIndexUrl() )
? baseUrl + "/.index"
: baseUrl + "/" + remoteRepository.getRemoteIndexUrl();
return baseUrl;
}
public TaskScheduler getTaskScheduler()
{
return taskScheduler;
}
public void setTaskScheduler( TaskScheduler taskScheduler )
{
this.taskScheduler = taskScheduler;
}
}

33
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexException.java Normal file
View File

@ -0,0 +1,33 @@
package org.apache.archiva.scheduler.indexing;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/**
* @author Olivier Lamy
* @since 1.4
*/
public class DownloadRemoteIndexException
extends Exception
{
public DownloadRemoteIndexException( String message, Throwable exception )
{
super( message, exception );
}
}

29
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexScheduler.java Normal file
View File

@ -0,0 +1,29 @@
package org.apache.archiva.scheduler.indexing;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/**
* @author Olivier Lamy
* @since 1.4
*/
public interface DownloadRemoteIndexScheduler
{
void scheduleDownloadRemote( String repositoryId, boolean now, boolean fullDownload )
throws DownloadRemoteIndexException;
}

373
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTask.java Normal file
View File

@ -0,0 +1,373 @@
package org.apache.archiva.scheduler.indexing;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.archiva.proxy.common.WagonFactoryException;
import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.io.FileUtils;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.updater.IndexUpdateRequest;
import org.apache.maven.index.updater.IndexUpdater;
import org.apache.maven.index.updater.ResourceFetcher;
import org.apache.maven.wagon.ConnectionException;
import org.apache.maven.wagon.ResourceDoesNotExistException;
import org.apache.maven.wagon.TransferFailedException;
import org.apache.maven.wagon.Wagon;
import org.apache.maven.wagon.authentication.AuthenticationException;
import org.apache.maven.wagon.authentication.AuthenticationInfo;
import org.apache.maven.wagon.authorization.AuthorizationException;
import org.apache.maven.wagon.events.TransferEvent;
import org.apache.maven.wagon.events.TransferListener;
import org.apache.maven.wagon.proxy.ProxyInfo;
import org.apache.maven.wagon.repository.Repository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
/**
* @author Olivier Lamy
* @since 1.4
*/
public class DownloadRemoteIndexTask
implements Runnable
{
private Logger log = LoggerFactory.getLogger( getClass() );
private RemoteRepository remoteRepository;
private NexusIndexer nexusIndexer;
private WagonFactory wagonFactory;
private NetworkProxy networkProxy;
private boolean fullDownload;
private List<String> runningRemoteDownloadIds;
private IndexUpdater indexUpdater;
public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest,
List<String> runningRemoteDownloadIds )
{
this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository();
this.nexusIndexer = downloadRemoteIndexTaskRequest.getNexusIndexer();
this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory();
this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy();
this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload();
this.runningRemoteDownloadIds = runningRemoteDownloadIds;
this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater();
}
public void run()
{
// so short lock : not sure we need it
synchronized ( this.runningRemoteDownloadIds )
{
if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) )
{
// skip it as it's running
log.info( "skip download index remote for repo {} it's already running",
this.remoteRepository.getId() );
return;
}
log.info( "start download remote index for remote repository " + this.remoteRepository.getId() );
this.runningRemoteDownloadIds.add( this.remoteRepository.getId() );
}
IndexingContext indexingContext =
nexusIndexer.getIndexingContexts().get( "remote-" + remoteRepository.getId() );
// TODO check if null ? normally not as created by DefaultDownloadRemoteIndexScheduler#startup
// create a temp directory to download files
final File tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" );
try
{
if ( tempIndexDirectory.exists() )
{
FileUtils.deleteDirectory( tempIndexDirectory );
}
tempIndexDirectory.mkdirs();
String baseIndexUrl = indexingContext.getIndexUpdateUrl();
final Wagon wagon = wagonFactory.getWagon( new URL( this.remoteRepository.getUrl() ).getProtocol() );
// TODO transferListener
wagon.addTransferListener( new DownloadListener() );
ProxyInfo proxyInfo = null;
if ( this.networkProxy != null )
{
proxyInfo = new ProxyInfo();
proxyInfo.setHost( this.networkProxy.getHost() );
proxyInfo.setPort( this.networkProxy.getPort() );
proxyInfo.setUserName( this.networkProxy.getUsername() );
proxyInfo.setPassword( this.networkProxy.getPassword() );
}
AuthenticationInfo authenticationInfo = null;
if ( this.remoteRepository.getUserName() != null )
{
authenticationInfo = new AuthenticationInfo();
authenticationInfo.setUserName( this.remoteRepository.getUserName() );
authenticationInfo.setPassword( this.remoteRepository.getPassword() );
}
wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo,
proxyInfo );
File indexDirectory = indexingContext.getIndexDirectoryFile();
if ( !indexDirectory.exists() )
{
indexDirectory.mkdirs();
}
/*
File[] indexFiles = indexDirectory.listFiles( new FileFilter()
{
public boolean accept( File file )
{
return !file.isDirectory();
}
} );
List<String> indexFileNames = new ArrayList<String>( indexFiles == null ? 0 : indexFiles.length );
for ( File f : indexFiles == null ? new File[0] : indexFiles )
{
indexFileNames.add( f.getName() );
}
*/
//List<String> files = wagon.getFileList( "" );
// take care about time stamp : no need to rebuild index
// TODO incremental honor fullDownload true !!
// FIXME dont fail all if one file fail ?
/*
for ( String file : files )
{
if ( !indexFileNames.contains( file ) && StringUtils.endsWith( file, ".gz" ) )
{
downloadFile( wagon, file, tempIndexDirectory );
File compressIndexUpdate = new File( tempIndexDirectory, file );
mergeCompressIndex( indexingContext, compressIndexUpdate, wagon );
}
}*/
ResourceFetcher resourceFetcher = new ResourceFetcher()
{
public void connect( String id, String url )
throws IOException
{
//no op
}
public void disconnect()
throws IOException
{
// no op
}
public InputStream retrieve( String name )
throws IOException, FileNotFoundException
{
try
{
log.debug( "resourceFetcher#retrieve, name:{}", name );
//TODO check those files are deleted !!
File file = new File( tempIndexDirectory, name );
if ( file.exists() )
{
file.delete();
}
//file.deleteOnExit();
wagon.get( name, file );
return new FileInputStream( file );
}
catch ( AuthorizationException e )
{
throw new IOException( e.getMessage(), e );
}
catch ( TransferFailedException e )
{
throw new IOException( e.getMessage(), e );
}
catch ( ResourceDoesNotExistException e )
{
throw new FileNotFoundException( e.getMessage() );
}
}
};
IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher );
this.indexUpdater.fetchAndUpdateIndex( request );
}
catch ( MalformedURLException e )
{
log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage(), e );
}
catch ( WagonFactoryException e )
{
log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage(), e );
}
catch ( ConnectionException e )
{
log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage(), e );
}
catch ( AuthenticationException e )
{
log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage(), e );
}
catch ( IOException e )
{
log.error( e.getMessage(), e );
throw new RuntimeException( e.getMessage(), e );
}
finally
{
//deleteDirectoryQuiet( tempIndexDirectory );
this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() );
}
log.info( "end download remote index for remote repository " + this.remoteRepository.getId() );
}
private void deleteDirectoryQuiet( File f )
{
try
{
FileUtils.deleteDirectory( f );
}
catch ( IOException e )
{
log.warn( "skip error delete " + f + ": " + e.getMessage() );
}
}
protected void mergeCompressIndex( IndexingContext context, final File compressedIndexUpdate, final Wagon wagon )
throws IOException, CompressorException
{
/*
final File tmpUncompressDirectory = new File( compressedIndexUpdate.getParent(),
StringUtils.substringBeforeLast( compressedIndexUpdate.getName(),
"." ) );
tmpUncompressDirectory.deleteOnExit();
final FileOutputStream fos =
new FileOutputStream( new File( tmpUncompressDirectory, compressedIndexUpdate.getName() ) );
try
{
if ( tmpUncompressDirectory.exists() )
{
tmpUncompressDirectory.delete();
}
tmpUncompressDirectory.mkdirs();
// gunzip the file to a directory and merge
// gunzip
final InputStream in = new FileInputStream( compressedIndexUpdate );
CompressorInputStream cis =
new CompressorStreamFactory().createCompressorInputStream( CompressorStreamFactory.GZIP, in );
IOUtils.copy( cis, fos );
in.close();
fos.flush();
fos.close();
// merge
}
finally
{
IOUtils.closeQuietly( fos );
//deleteDirectoryQuiet( tmpUncompressDirectory );
//FileUtils.deleteQuietly( tmpUncompressDirectory );
}
*/
}
protected void downloadFile( Wagon wagon, String file, File tempIndexDirectory )
{
try
{
wagon.get( file, new File( tempIndexDirectory, file ) );
}
catch ( Exception e )
{
log.warn( "skip fail to download " + file + ": " + e.getMessage() );
}
}
public static class DownloadListener
implements TransferListener
{
private Logger log = LoggerFactory.getLogger( getClass() );
public void transferInitiated( TransferEvent transferEvent )
{
log.debug( "initiate transfer of {}", transferEvent.getResource().getName() );
}
public void transferStarted( TransferEvent transferEvent )
{
log.debug( "start transfer of {}", transferEvent.getResource().getName() );
}
public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
{
log.debug( "transfer of {} : {}/{}",
Arrays.asList( transferEvent.getResource().getName(), buffer.length, length ).toArray() );
}
public void transferCompleted( TransferEvent transferEvent )
{
log.info( "end of transfer file " + transferEvent.getResource().getName() );
}
public void transferError( TransferEvent transferEvent )
{
log.info( "error of transfer file {}: {}", Arrays.asList( transferEvent.getResource().getName(),
transferEvent.getException().getMessage() ).toArray(
new Object[2] ), transferEvent.getException() );
}
public void debug( String message )
{
log.debug( "transfer debug {}", message );
}
}
}

115
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskRequest.java Normal file
View File

@ -0,0 +1,115 @@
package org.apache.archiva.scheduler.indexing;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.archiva.admin.model.beans.NetworkProxy;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.proxy.common.WagonFactory;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.updater.IndexUpdater;
/**
* @author Olivier Lamy
* @since 1.4
*/
public class DownloadRemoteIndexTaskRequest
{
private RemoteRepository remoteRepository;
private NexusIndexer nexusIndexer;
private WagonFactory wagonFactory;
private NetworkProxy networkProxy;
private boolean fullDownload;
private IndexUpdater indexUpdater;
public DownloadRemoteIndexTaskRequest()
{
// no op
}
public RemoteRepository getRemoteRepository()
{
return remoteRepository;
}
public DownloadRemoteIndexTaskRequest setRemoteRepository( RemoteRepository remoteRepository )
{
this.remoteRepository = remoteRepository;
return this;
}
public NexusIndexer getNexusIndexer()
{
return nexusIndexer;
}
public DownloadRemoteIndexTaskRequest setNexusIndexer( NexusIndexer nexusIndexer )
{
this.nexusIndexer = nexusIndexer;
return this;
}
public WagonFactory getWagonFactory()
{
return wagonFactory;
}
public DownloadRemoteIndexTaskRequest setWagonFactory( WagonFactory wagonFactory )
{
this.wagonFactory = wagonFactory;
return this;
}
public NetworkProxy getNetworkProxy()
{
return networkProxy;
}
public DownloadRemoteIndexTaskRequest setNetworkProxy( NetworkProxy networkProxy )
{
this.networkProxy = networkProxy;
return this;
}
public boolean isFullDownload()
{
return fullDownload;
}
public DownloadRemoteIndexTaskRequest setFullDownload( boolean fullDownload )
{
this.fullDownload = fullDownload;
return this;
}
public IndexUpdater getIndexUpdater()
{
return indexUpdater;
}
public DownloadRemoteIndexTaskRequest setIndexUpdater( IndexUpdater indexUpdater )
{
this.indexUpdater = indexUpdater;
return this;
}
}

19
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/resources/META-INF/spring-context.xml
View File

@ -20,25 +20,38 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:task="http://www.springframework.org/schema/task"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd"
default-lazy-init="true">
http://www.springframework.org/schema/context/spring-context-3.0.xsd
http://www.springframework.org/schema/task
http://www.springframework.org/schema/task/spring-task-3.0.xsd"
default-lazy-init="false">
<context:annotation-config/>
<context:component-scan base-package="org.apache.archiva.scheduler.indexing"/>
<bean id="logger" class="org.apache.archiva.common.utils.Slf4JPlexusLogger">
<constructor-arg type="java.lang.Class"><value>org.sonatype.nexus.index.DefaultNexusIndexer</value></constructor-arg>
<constructor-arg type="java.lang.Class">
<value>org.apache.maven.index.DefaultNexusIndexer</value>
</constructor-arg>
</bean>
<bean name="taskQueue#indexing" class="org.codehaus.plexus.taskqueue.DefaultTaskQueue"/>
<bean name="taskQueueExecutor#indexing" class="org.codehaus.plexus.taskqueue.execution.ThreadedTaskQueueExecutor">
<property name="executor" ref="taskExecutor#indexing"/>
<property name="queue" ref="taskQueue#indexing"/>
<property name="name" value="indexing"/>
</bean>
<bean name="taskScheduler#indexDownloadRemote"
class="org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler">
<property name="poolSize" value="4"/>
<property name="threadGroupName" value="indexDownloadRemote"/>
</bean>
</beans>

BIN
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/index-updates/nexus-maven-repository-index.gz Normal file
View File

Binary file not shown.

36
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/index-updates/nexus-maven-repository-index.properties Normal file
View File

@ -0,0 +1,36 @@
#Sun Sep 25 09:15:00 CEST 2011
nexus.index.id=test-repo
nexus.index.chain-id=1316094851802
nexus.index.timestamp=20110925071457.200 +0000
nexus.index.incremental-19=39
nexus.index.incremental-18=40
nexus.index.incremental-17=41
nexus.index.incremental-16=42
nexus.index.incremental-15=43
nexus.index.incremental-14=44
nexus.index.incremental-13=45
nexus.index.incremental-9=49
nexus.index.incremental-12=46
nexus.index.incremental-8=50
nexus.index.incremental-11=47
nexus.index.incremental-7=51
nexus.index.incremental-10=48
nexus.index.incremental-6=52
nexus.index.incremental-5=53
nexus.index.incremental-4=54
nexus.index.incremental-3=55
nexus.index.incremental-2=56
nexus.index.last-incremental=58
nexus.index.incremental-1=57
nexus.index.incremental-0=58
nexus.index.incremental-29=29
nexus.index.incremental-28=30
nexus.index.incremental-27=31
nexus.index.incremental-26=32
nexus.index.incremental-25=33
nexus.index.incremental-24=34
nexus.index.time=20110925071457.200 +0000
nexus.index.incremental-23=35
nexus.index.incremental-22=36
nexus.index.incremental-21=37
nexus.index.incremental-20=38

BIN
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/index-updates/nexus-maven-repository-index.zip Normal file
View File

Binary file not shown.

165
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/java/org/apache/archiva/scheduler/indexing/DownloadRemoteIndexTaskTest.java Normal file
View File

@ -0,0 +1,165 @@
package org.apache.archiva.scheduler.indexing;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import junit.framework.TestCase;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
import org.apache.archiva.common.utils.FileUtil;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.maven.index.FlatSearchRequest;
import org.apache.maven.index.FlatSearchResponse;
import org.apache.maven.index.MAVEN;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.expr.StringSearchExpression;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.DefaultServlet;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
/**
* @author Olivier Lamy
*/
@RunWith( SpringJUnit4ClassRunner.class )
@ContextConfiguration( locations = { "classpath*:/META-INF/spring-context.xml", "classpath*:/spring-context.xml" } )
public class DownloadRemoteIndexTaskTest
extends TestCase
{
private Server server;
private int port;
private Logger log = LoggerFactory.getLogger( getClass() );
@Inject
RemoteRepositoryAdmin remoteRepositoryAdmin;
@Inject
DefaultDownloadRemoteIndexScheduler downloadRemoteIndexScheduler;
@Inject
PlexusSisuBridge plexusSisuBridge;
NexusIndexer nexusIndexer;
@Before
public void initialize()
throws Exception
{
super.setUp();
server = new Server( 0 );
createContext( server, new File( "src/test/" ) );
this.server.start();
Connector connector = this.server.getConnectors()[0];
this.port = connector.getLocalPort();
log.info( "start server on port " + this.port );
nexusIndexer = plexusSisuBridge.lookup( NexusIndexer.class );
}
protected void createContext( Server server, File repositoryDirectory )
throws IOException
{
ServletContextHandler context = new ServletContextHandler();
context.setResourceBase( repositoryDirectory.getAbsolutePath() );
context.setContextPath( "/" );
ServletHolder sh = new ServletHolder( DefaultServlet.class );
context.addServlet( sh, "/" );
server.setHandler( context );
}
@After
public void tearDown()
throws Exception
{
server.stop();
super.tearDown();
}
@Test
public void downloadAndMergeRemoteIndexInEmptyIndex()
throws Exception
{
RemoteRepository remoteRepository = getRemoteRepository();
remoteRepositoryAdmin.addRemoteRepository( remoteRepository, null );
downloadRemoteIndexScheduler.startup();
downloadRemoteIndexScheduler.scheduleDownloadRemote( "test-repo", true, true );
( (ThreadPoolTaskScheduler) downloadRemoteIndexScheduler.getTaskScheduler() ).getScheduledExecutor().awaitTermination(
10, TimeUnit.SECONDS );
remoteRepositoryAdmin.deleteRemoteRepository( "test-repo", null );
// search
BooleanQuery iQuery = new BooleanQuery();
iQuery.add( nexusIndexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( "commons-logging" ) ),
BooleanClause.Occur.SHOULD );
FlatSearchRequest rq = new FlatSearchRequest( iQuery );
rq.setContexts(
Arrays.asList( nexusIndexer.getIndexingContexts().get( "remote-" + getRemoteRepository().getId() ) ) );
FlatSearchResponse response = nexusIndexer.searchFlat( rq );
log.info( "returned hit count:" + response.getReturnedHitsCount() );
assertEquals( 8, response.getReturnedHitsCount() );
}
protected RemoteRepository getRemoteRepository()
{
RemoteRepository remoteRepository = new RemoteRepository();
File indexDirectory =
new File( FileUtil.getBasedir(), "target/index/test-" + Long.toString( System.currentTimeMillis() ) );
indexDirectory.mkdirs();
indexDirectory.deleteOnExit();
remoteRepository.setName( "foo" );
remoteRepository.setIndexDirectory( indexDirectory.getAbsolutePath() );
remoteRepository.setDownloadRemoteIndex( true );
remoteRepository.setId( "test-repo" );
remoteRepository.setUrl( "http://localhost:" + port );
remoteRepository.setRemoteIndexUrl( "http://localhost:" + port + "/index-updates/" );
return remoteRepository;
}
}

228
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/resources/META-INF/redback/redback-core.xml Normal file
View File

@ -0,0 +1,228 @@
<redback-role-model>
<modelVersion>1.0.0</modelVersion>
<applications>
<application>
<id>System</id>
<description>Roles that apply system-wide, across all of the applications</description>
<version>1.0.0</version>
<resources>
<resource>
<id>global</id>
<name>*</name>
<permanent>true</permanent>
<description>global resource implies full access for authorization</description>
</resource>
<resource>
<id>username</id>
<name>${username}</name>
<permanent>true</permanent>
<description>replaced with the username of the principal at authorization check time</description>
</resource>
</resources>
<operations>
<operation>
<id>configuration-edit</id>
<name>configuration-edit</name>
<description>edit configuration</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-user-create</id>
<name>user-management-user-create</name>
<description>create user</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-user-edit</id>
<name>user-management-user-edit</name>
<description>edit user</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-user-role</id>
<name>user-management-user-role</name>
<description>user roles</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-user-delete</id>
<name>user-management-user-delete</name>
<description>delete user</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-user-list</id>
<name>user-management-user-list</name>
<description>list users</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-role-grant</id>
<name>user-management-role-grant</name>
<description>grant role</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-role-drop</id>
<name>user-management-role-drop</name>
<description>drop role</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-rbac-admin</id>
<name>user-management-rbac-admin</name>
<description>administer rbac</description>
<permanent>true</permanent>
</operation>
<operation>
<id>guest-access</id>
<name>guest-access</name>
<description>access guest</description>
<permanent>true</permanent>
</operation>
<operation>
<id>user-management-manage-data</id>
<name>user-management-manage-data</name>
<description>manage data</description>
<permanent>true</permanent>
</operation>
</operations>
<roles>
<role>
<id>system-administrator</id>
<name>System Administrator</name>
<permanent>true</permanent>
<assignable>true</assignable>
<permissions>
<permission>
<id>edit-redback-configuration</id>
<name>Edit Redback Configuration</name>
<operation>configuration-edit</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>manage-rbac-setup</id>
<name>User RBAC Management</name>
<operation>user-management-rbac-admin</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>manage-rbac-data</id>
<name>RBAC Manage Data</name>
<operation>user-management-manage-data</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
</permissions>
<childRoles>
<childRole>user-administrator</childRole>
</childRoles>
</role>
<role>
<id>user-administrator</id>
<name>User Administrator</name>
<permanent>true</permanent>
<assignable>true</assignable>
<permissions>
<permission>
<id>drop-roles-for-anyone</id>
<name>Drop Roles for Anyone</name>
<operation>user-management-role-drop</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>grant-roles-for-anyone</id>
<name>Grant Roles for Anyone</name>
<operation>user-management-role-grant</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>user-create</id>
<name>Create Users</name>
<operation>user-management-user-create</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>user-delete</id>
<name>Delete Users</name>
<operation>user-management-user-delete</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>user-edit</id>
<name>Edit Users</name>
<operation>user-management-user-edit</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>access-users-roles</id>
<name>Access Users Roles</name>
<operation>user-management-user-role</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
<permission>
<id>access-user-list</id>
<name>Access User List</name>
<operation>user-management-user-list</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
</permissions>
</role>
<role>
<id>edit-users-list</id>
<name>edit users list</name>
<permanent>true</permanent>
<assignable>true</assignable>
<permissions>
<permission>
<id>access-user-list</id>
<name>Access User List</name>
<operation>user-management-user-list</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
</permissions>
</role>
<role>
<id>registered-user</id>
<name>Registered User</name>
<permanent>true</permanent>
<assignable>true</assignable>
<permissions>
<permission>
<id>edit-user-by-username</id>
<name>Edit User Data by Username</name>
<operation>user-management-user-edit</operation>
<resource>username</resource>
<permanent>true</permanent>
</permission>
</permissions>
</role>
<role>
<id>guest</id>
<name>Guest</name>
<permanent>true</permanent>
<assignable>true</assignable>
<permissions>
<permission>
<id>guest-permission</id>
<name>Guest Permission</name>
<operation>guest-access</operation>
<resource>global</resource>
<permanent>true</permanent>
</permission>
</permissions>
</role>
</roles>
</application>
</applications>
</redback-role-model>

44
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/resources/log4j.xml Normal file
View File

@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d [%t] %-5p %c %x - %m%n"/>
</layout>
</appender>
<logger name="org.springframework">
<level value="ERROR"/>
</logger>
<logger name="org.apache.archiva.scheduler.indexing">
<level value="debug"/>
</logger>
<root>
<priority value ="info" />
<appender-ref ref="console" />
</root>
</log4j:configuration>

62
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/test/resources/spring-context.xml Normal file
View File

@ -0,0 +1,62 @@
<?xml version="1.0"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd" default-lazy-init="true">
<bean name="scheduler" class="org.codehaus.redback.components.scheduler.DefaultScheduler">
<property name="properties">
<props>
<prop key="org.quartz.scheduler.instanceName">scheduler1</prop>
<prop key="org.quartz.threadPool.class">org.quartz.simpl.SimpleThreadPool</prop>
<prop key="org.quartz.threadPool.threadCount">2</prop>
<prop key="org.quartz.threadPool.threadPriority">4</prop>
<prop key="org.quartz.jobStore.class">org.quartz.simpl.RAMJobStore</prop>
</props>
</property>
</bean>
<!-- wire up more basic configuration so it doesn't overwrite any config files -->
<bean name="archivaConfiguration#default" class="org.apache.archiva.configuration.DefaultArchivaConfiguration">
<property name="registry" ref="registry#default"/>
</bean>
<alias name="archivaConfiguration#default" alias="archivaConfiguration"/>
<bean name="registry#default" class="org.codehaus.redback.components.registry.commons.CommonsConfigurationRegistry">
<property name="properties">
<value>
<![CDATA[
<configuration>
<system/>
<xml fileName="${appserver.base}/conf/archiva.xml" config-forceCreate="true"
config-optional="true"
config-name="org.apache.maven.archiva.base" config-at="org.apache.maven.archiva"/>
</configuration>
]]>
</value>
</property>
</bean>
</beans>

2
archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/main/java/org/apache/archiva/scheduler/repository/ArchivaRepositoryScanningTaskExecutor.java
View File

@ -53,7 +53,7 @@ import java.util.Date;
*/
@Service( "taskExecutor#repository-scanning" )
public class ArchivaRepositoryScanningTaskExecutor
implements TaskExecutor, Initializable
implements TaskExecutor
{
private Logger log = LoggerFactory.getLogger( ArchivaRepositoryScanningTaskExecutor.class );

13
archiva-modules/archiva-scheduler/archiva-scheduler-repository/src/test/java/org/apache/archiva/mock/MockArchivaAdministration.java
View File

@ -24,6 +24,7 @@ import org.apache.archiva.admin.model.admin.ArchivaAdministration;
import org.apache.archiva.admin.model.beans.FileType;
import org.apache.archiva.admin.model.beans.LegacyArtifactPath;
import org.apache.archiva.admin.model.beans.OrganisationInformation;
import org.apache.archiva.admin.model.beans.UiConfiguration;
import org.apache.archiva.configuration.ArchivaConfiguration;
import java.util.ArrayList;
@ -162,4 +163,16 @@ public class MockArchivaAdministration
{
this.archivaConfiguration = archivaConfiguration;
}
public UiConfiguration getUiConfiguration()
throws RepositoryAdminException
{
return null; //To change body of implemented methods use File | Settings | File Templates.
}
public void updateUiConfiguration( UiConfiguration uiConfiguration )
throws RepositoryAdminException
{
//To change body of implemented methods use File | Settings | File Templates.
}
}

1
archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml
View File

@ -252,6 +252,7 @@
<systemPropertyVariables>
<plexus.home>${project.build.outputDirectory}</plexus.home>
<appserver.base>${basedir}/target/appserver-base</appserver.base>
<java.io.tmpdir>${project.build.outputDirectory}</java.io.tmpdir>
</systemPropertyVariables>
</configuration>
</plugin>

6
archiva-modules/archiva-web/archiva-webapp/pom.xml
View File

@ -171,6 +171,10 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-bundle-jaxrs</artifactId>
@ -518,7 +522,7 @@
<dependentWarExcludes>META-INF/**,WEB-INF/web.xml,WEB-INF/classes/xwork.xml,WEB-INF/lib/**
</dependentWarExcludes>
<warSourceExcludes>
WEB-INF/lib/xalan-*.jar,WEB-INF/lib/velocity-dep-*.jar,WEB-INF/lib/xml-apis-*.jar,WEB-INF/lib/wstx-asl-*.jar,WEB-INF/lib/stax-utils-*.jar,WEB-INF/lib/xercesImpl-*.jar
WEB-INF/lib/xalan-*.jar,WEB-INF/lib/velocity-dep-*.jar,WEB-INF/lib/xml-apis-*.jar,WEB-INF/lib/wstx-asl-*.jar,WEB-INF/lib/stax-utils-*.jar,WEB-INF/lib/xercesImpl-*.jar,WEB-INF/lib/commons-lang-*.jar
</warSourceExcludes>
</configuration>
</plugin>

2
archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/archiva/web/action/admin/connectors/proxy/ProxyConnectorsAction.java
View File

@ -20,7 +20,7 @@ package org.apache.archiva.web.action.admin.connectors.proxy;
*/
import com.opensymphony.xwork2.Preparable;
import org.apache.archiva.admin.model.AbstractRepository;
import org.apache.archiva.admin.model.beans.AbstractRepository;
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.ProxyConnector;
import org.springframework.context.annotation.Scope;

43
archiva-modules/archiva-web/archiva-webapp/src/main/java/org/apache/archiva/web/action/admin/repositories/EditRemoteRepositoryAction.java
View File

@ -22,10 +22,14 @@ package org.apache.archiva.web.action.admin.repositories;
import com.opensymphony.xwork2.Preparable;
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.scheduler.indexing.DownloadRemoteIndexException;
import org.apache.archiva.scheduler.indexing.DownloadRemoteIndexScheduler;
import org.apache.commons.lang.StringUtils;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Controller;
import javax.inject.Inject;
/**
* EditRemoteRepositoryAction
*
@ -47,6 +51,11 @@ public class EditRemoteRepositoryAction
*/
private String repoid;
private boolean now, fullDownload;
@Inject
private DownloadRemoteIndexScheduler downloadRemoteIndexScheduler;
public void prepare()
throws RepositoryAdminException
{
@ -83,6 +92,20 @@ public class EditRemoteRepositoryAction
return result;
}
public String downloadRemoteIndex()
{
try
{
downloadRemoteIndexScheduler.scheduleDownloadRemote( repoid, now, fullDownload );
}
catch ( DownloadRemoteIndexException e )
{
addActionError( "DownloadRemoteIndexException: " + e.getMessage() );
return INPUT;
}
return SUCCESS;
}
public RemoteRepository getRepository()
{
return repository;
@ -102,4 +125,24 @@ public class EditRemoteRepositoryAction
{
this.repoid = repoid;
}
public boolean isNow()
{
return now;
}
public void setNow( boolean now )
{
this.now = now;
}
public boolean isFullDownload()
{
return fullDownload;
}
public void setFullDownload( boolean fullDownload )
{
this.fullDownload = fullDownload;
}
}

6
archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/editRemoteRepository.jsp
View File

@ -40,6 +40,12 @@
<%@ include file="/WEB-INF/jsp/admin/include/remoteRepositoryForm.jspf" %>
<s:submit value="Update Repository"/>
</s:form>
<s:form method="post" action="editRemoteRepository!downloadRemoteIndex" namespace="/admin" validate="false">
<s:hidden name="repoid"/>
<s:checkbox name="now" label="Now" />
<s:checkbox name="fullDownload" label="Full download"/>
<s:submit value="download Remote Index"/>
</s:form>
<script type="text/javascript">
document.getElementById("editRemoteRepository_repository_name").focus();

8
archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/jsp/admin/include/remoteRepositoryForm.jspf
View File

@ -21,9 +21,15 @@
<%@ taglib prefix="s" uri="/struts-tags" %>
<s:textfield name="repository.name" label="Name" size="50" required="true"/>
<s:textfield name="repository.url" label="URL" size="50" required="true"/>
<s:textfield name="repository.url" label="URL" size="60" required="true"/>
<s:textfield name="repository.userName" label="Username" size="25" required="false"/>
<s:password name="repository.password" label="Password" size="25" required="false"/>
<s:textfield name="repository.timeout" label="Timeout in seconds" size="3" required="false"/>
<s:checkbox name="repository.downloadRemoteIndex" label="Activate download remote index" />
<s:textfield name="repository.remoteIndexUrl" label="Remote index url, can be relative to url" size="60" required="false"/>
<s:textfield name="repository.cronExpression" label="Cron expression" size="10" required="false"/>
<s:textfield name="repository.indexDirectory" label="Directory index storage" size="60" required="false"/>
<s:select list="#@java.util.LinkedHashMap@{'default' : 'Maven 2.x Repository', 'legacy' : 'Maven 1.x Repository'}"
name="repository.layout" label="Type"/>

5
pom.xml
View File

@ -537,6 +537,11 @@
<artifactId>commons-lang</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>