Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding context merge to provider

This commit is contained in:
Martin Stockhammer 2019-06-22 03:43:52 +02:00
parent 380c152114
commit a52b7f2005
13 changed files with 450 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java
View File

@ -29,6 +29,9 @@ import org.apache.archiva.configuration.Configuration;
import org.apache.archiva.configuration.RepositoryGroupConfiguration;
import org.apache.archiva.metadata.model.facets.AuditEvent;
import org.apache.archiva.indexer.merger.MergedRemoteIndexesScheduler;
import org.apache.archiva.repository.EditableRepository;
import org.apache.archiva.repository.EditableRepositoryGroup;
import org.apache.archiva.repository.RepositoryException;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.archiva.repository.features.IndexCreationFeature;
import org.apache.commons.lang.StringUtils;
@ -38,6 +41,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -70,6 +74,7 @@ public class DefaultRepositoryGroupAdmin
private ManagedRepositoryAdmin managedRepositoryAdmin;
@Inject
@Named("mergedRemoteIndexesScheduler#default")
private MergedRemoteIndexesScheduler mergedRemoteIndexesScheduler;
@Inject
@ -113,16 +118,12 @@ public class DefaultRepositoryGroupAdmin
}
@Override
public List<RepositoryGroup> getRepositoriesGroups()
throws RepositoryAdminException
{
public List<RepositoryGroup> getRepositoriesGroups() {
return repositoryRegistry.getRepositoryGroups().stream().map( r -> convertRepositoryGroupObject( r ) ).collect( Collectors.toList());
}
@Override
public RepositoryGroup getRepositoryGroup( String repositoryGroupId )
throws RepositoryAdminException
{
public RepositoryGroup getRepositoryGroup( String repositoryGroupId ) {
return convertRepositoryGroupObject( repositoryRegistry.getRepositoryGroup( repositoryGroupId ) );
}
@ -138,10 +139,14 @@ public class DefaultRepositoryGroupAdmin
repositoryGroupConfiguration.setRepositories( repositoryGroup.getRepositories() );
repositoryGroupConfiguration.setMergedIndexPath( repositoryGroup.getMergedIndexPath() );
repositoryGroupConfiguration.setMergedIndexTtl( repositoryGroup.getMergedIndexTtl() );
repositoryGroupConfiguration.setCronExpression( repositoryGroup.getCronExpression() );
Configuration configuration = getArchivaConfiguration().getConfiguration();
configuration.addRepositoryGroup( repositoryGroupConfiguration );
saveConfiguration( configuration );
repositoryGroupConfiguration.setCronExpression( StringUtils.isEmpty(repositoryGroup.getCronExpression()) ? "0 0 03 ? * MON" : repositoryGroup.getCronExpression() );
try {
repositoryRegistry.putRepositoryGroup(repositoryGroupConfiguration);
} catch (RepositoryException e) {
e.printStackTrace();
}
triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.ADD_REPO_GROUP, auditInformation );
mergedRemoteIndexesScheduler.schedule( repositoryRegistry.getRepositoryGroup( repositoryGroup.getId()), getMergedIndexDirectory( repositoryGroup.getId() ) );
return Boolean.TRUE;
@ -151,17 +156,16 @@ public class DefaultRepositoryGroupAdmin
public Boolean deleteRepositoryGroup( String repositoryGroupId, AuditInformation auditInformation )
throws RepositoryAdminException
{
Configuration configuration = getArchivaConfiguration().getConfiguration();
RepositoryGroupConfiguration repositoryGroupConfiguration =
configuration.getRepositoryGroupsAsMap().get( repositoryGroupId );
org.apache.archiva.repository.RepositoryGroup repositoryGroup = repositoryRegistry.getRepositoryGroup(repositoryGroupId);
try {
repositoryRegistry.removeRepositoryGroup(repositoryGroup);
} catch (RepositoryException e) {
log.error("Removal of repository group {} failed: {}", repositoryGroup.getId(), e.getMessage(), e);
throw new RepositoryAdminException("Removal of repository failed: " + e.getMessage(), e);
}
mergedRemoteIndexesScheduler.unschedule(
repositoryRegistry.getRepositoryGroup( repositoryGroupId ) );
if ( repositoryGroupConfiguration == null )
{
throw new RepositoryAdminException(
"repositoryGroup with id " + repositoryGroupId + " doesn't not exists so cannot remove" );
}
configuration.removeRepositoryGroup( repositoryGroupConfiguration );
triggerAuditEvent( repositoryGroupId, null, AuditEvent.DELETE_REPO_GROUP, auditInformation );
return Boolean.TRUE;
@ -180,24 +184,23 @@ public class DefaultRepositoryGroupAdmin
{
validateRepositoryGroup( repositoryGroup, true );
validateManagedRepositoriesExists( repositoryGroup.getRepositories() );
Configuration configuration = getArchivaConfiguration().getConfiguration();
RepositoryGroupConfiguration repositoryGroupConfiguration =
configuration.getRepositoryGroupsAsMap().get( repositoryGroup.getId() );
configuration.removeRepositoryGroup( repositoryGroupConfiguration );
repositoryGroupConfiguration.setRepositories( repositoryGroup.getRepositories() );
repositoryGroupConfiguration.setMergedIndexPath( repositoryGroup.getMergedIndexPath() );
repositoryGroupConfiguration.setMergedIndexTtl( repositoryGroup.getMergedIndexTtl() );
repositoryGroupConfiguration.setCronExpression( repositoryGroup.getCronExpression() );
configuration.addRepositoryGroup( repositoryGroupConfiguration );
saveConfiguration( configuration );
if ( triggerAuditEvent )
{
triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.MODIFY_REPO_GROUP, auditInformation );
try {
repositoryRegistry.putRepositoryGroup(repositoryGroupConfiguration);
} catch (RepositoryException e) {
e.printStackTrace();
}
org.apache.archiva.repository.RepositoryGroup rg = repositoryRegistry.getRepositoryGroup( repositoryGroup.getId( ) );
mergedRemoteIndexesScheduler.unschedule( rg );
mergedRemoteIndexesScheduler.schedule( rg, getMergedIndexDirectory( repositoryGroup.getId() ) );
@ -210,22 +213,33 @@ public class DefaultRepositoryGroupAdmin
AuditInformation auditInformation )
throws RepositoryAdminException
{
RepositoryGroup repositoryGroup = getRepositoryGroup( repositoryGroupId );
org.apache.archiva.repository.RepositoryGroup repositoryGroup = repositoryRegistry.getRepositoryGroup( repositoryGroupId );
if ( repositoryGroup == null )
{
throw new RepositoryAdminException(
"repositoryGroup with id " + repositoryGroupId + " doesn't not exists so cannot add repository to it" );
"repositoryGroup with id " + repositoryGroupId + " doesn't not exists so cannot add repository to it" );
}
if ( repositoryGroup.getRepositories().contains( repositoryId ) )
if (!(repositoryGroup instanceof EditableRepositoryGroup)) {
throw new RepositoryAdminException("The repository group is not editable "+repositoryGroupId);
}
EditableRepositoryGroup editableRepositoryGroup = (EditableRepositoryGroup) repositoryGroup;
if ( editableRepositoryGroup.getRepositories().stream().anyMatch( repo -> repositoryId.equals(repo.getId())) )
{
throw new RepositoryAdminException(
"repositoryGroup with id " + repositoryGroupId + " already contain repository with id" + repositoryId );
}
validateManagedRepositoriesExists( Arrays.asList( repositoryId ) );
org.apache.archiva.repository.ManagedRepository managedRepo = repositoryRegistry.getManagedRepository(repositoryId);
if (managedRepo==null) {
throw new RepositoryAdminException("Repository with id "+repositoryId+" does not exist" );
}
repositoryGroup.addRepository( repositoryId );
updateRepositoryGroup( repositoryGroup, auditInformation, false );
editableRepositoryGroup.addRepository( managedRepo );
try {
repositoryRegistry.putRepositoryGroup(editableRepositoryGroup);
} catch (RepositoryException e) {
throw new RepositoryAdminException("Could not store the repository group "+repositoryGroupId, e);
}
triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.ADD_REPO_TO_GROUP, auditInformation );
return Boolean.TRUE;
}
@ -235,23 +249,31 @@ public class DefaultRepositoryGroupAdmin
AuditInformation auditInformation )
throws RepositoryAdminException
{
RepositoryGroup repositoryGroup = getRepositoryGroup( repositoryGroupId );
org.apache.archiva.repository.RepositoryGroup repositoryGroup = repositoryRegistry.getRepositoryGroup( repositoryGroupId );
if ( repositoryGroup == null )
{
throw new RepositoryAdminException( "repositoryGroup with id " + repositoryGroupId
+ " doesn't not exists so cannot remove repository from it" );
}
if ( !repositoryGroup.getRepositories().contains( repositoryId ) )
if ( !repositoryGroup.getRepositories().stream().anyMatch( repo -> repositoryId.equals(repo.getId()) ) )
{
throw new RepositoryAdminException(
"repositoryGroup with id " + repositoryGroupId + " doesn't not contains repository with id"
+ repositoryId
);
}
if (!(repositoryGroup instanceof EditableRepositoryGroup)) {
throw new RepositoryAdminException("Repository group is not editable " + repositoryGroupId);
}
EditableRepositoryGroup editableRepositoryGroup = (EditableRepositoryGroup) repositoryGroup;
repositoryGroup.removeRepository( repositoryId );
updateRepositoryGroup( repositoryGroup, auditInformation, false );
editableRepositoryGroup.removeRepository( repositoryId );
try {
repositoryRegistry.putRepositoryGroup(editableRepositoryGroup);
} catch (RepositoryException e) {
throw new RepositoryAdminException("Could not store repository group " + repositoryGroupId, e);
}
triggerAuditEvent( repositoryGroup.getId(), null, AuditEvent.DELETE_REPO_FROM_GROUP, auditInformation );
return Boolean.TRUE;
}

5
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/test/java/org/apache/archiva/admin/mock/ArchivaIndexManagerMock.java
View File

@ -519,6 +519,11 @@ public class ArchivaIndexManagerMock implements ArchivaIndexManager {
}
}
@Override
public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts, boolean packIndex) throws UnsupportedOperationException, IndexCreationFailedException {
return null;
}
private StorageAsset getIndexPath( Repository repo) throws IOException {
IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();
Path repoDir = repo.getLocalPath();

19
archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexManager.java
View File

@ -21,9 +21,12 @@ package org.apache.archiva.indexer;
import org.apache.archiva.repository.Repository;
import org.apache.archiva.repository.RepositoryType;
import org.apache.archiva.repository.content.StorageAsset;
import java.net.URI;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
public interface ArchivaIndexManager {
@ -99,5 +102,19 @@ public interface ArchivaIndexManager {
* Updates the local path where the index is stored using the repository information.
* @return
*/
public void updateLocalIndexPath(Repository repo);
void updateLocalIndexPath(Repository repo);
/**
* Merges a list of contexts into a single one.
*
* @param destinationRepo The destination repository
* @param contexts The contexts of the indexes that should be merged.
* @param packIndex True, if the merged index should be packed, otherwise false.
* @return The merged context
* @throws UnsupportedOperationException if the underlying implementation does not allow to merge indexing contexts
*/
ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
boolean packIndex) throws UnsupportedOperationException,
IndexCreationFailedException;
}

94
archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/indexer/merger/BasicIndexMerger.java → archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java
View File

@ -1,5 +1,4 @@
package org.apache.archiva.indexer.merger;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -9,7 +8,7 @@ package org.apache.archiva.indexer.merger;
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@ -20,12 +19,20 @@ package org.apache.archiva.indexer.merger;
*/
import org.apache.archiva.common.utils.FileUtils;
import org.apache.archiva.indexer.ArchivaIndexManager;
import org.apache.archiva.indexer.ArchivaIndexingContext;
import org.apache.archiva.indexer.IndexCreationFailedException;
import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.merger.IndexMergerRequest;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.apache.archiva.repository.Repository;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.commons.lang.time.StopWatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
import java.io.IOException;
@ -33,32 +40,37 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;
/**
* @author Martin Stockhammer <martin_s@apache.org>
* @author Olivier Lamy
* @since 1.4-M2
*/
public class BasicIndexMerger implements IndexMerger
@Service("indexMerger#default")
public class DefaultIndexMerger
implements IndexMerger
{
@Inject
RepositoryRegistry repositoryRegistry;
private Logger log = LoggerFactory.getLogger( getClass() );
private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<>();
private List<ArchivaIndexingContext> temporaryContextes = new CopyOnWriteArrayList<>( );
private List<String> runningGroups = new CopyOnWriteArrayList<>();
@Inject
public BasicIndexMerger( )
public DefaultIndexMerger( )
{
}
@Override
public ArchivaIndexingContext buildMergedIndex( IndexMergerRequest indexMergerRequest )
public ArchivaIndexingContext buildMergedIndex(IndexMergerRequest indexMergerRequest )
throws IndexMergerException
{
String groupId = indexMergerRequest.getGroupId();
@ -70,49 +82,35 @@ public class BasicIndexMerger implements IndexMerger
}
runningGroups.add( groupId );
StopWatch stopWatch = new StopWatch();
stopWatch.reset();
stopWatch.start();
try {
stopWatch.reset();
stopWatch.start();
Path mergedIndexDirectory = indexMergerRequest.getMergedIndexDirectory();
Path mergedIndexDirectory = indexMergerRequest.getMergedIndexDirectory();
Repository destinationRepository = repositoryRegistry.getRepository(indexMergerRequest.getGroupId());
String tempRepoId = mergedIndexDirectory.getFileName().toString();
try
{
Path indexLocation = mergedIndexDirectory.resolve( indexMergerRequest.getMergedIndexPath() );
List<ArchivaIndexingContext> members = indexMergerRequest.getRepositoriesIds( ).stream( ).map( id ->
repositoryRegistry.getRepository( id ) )
.map( repo -> repo.getIndexingContext() ).filter( Objects::nonNull ).collect( Collectors.toList() );
members.get( 0 ).
if ( indexMergerRequest.isPackIndex() )
{
IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
mergedCtx.acquireIndexSearcher().getIndexReader(), //
indexLocation.toFile() );
indexPacker.packIndex( request );
ArchivaIndexManager idxManager = repositoryRegistry.getIndexManager(destinationRepository.getType());
List<ArchivaIndexingContext> sourceContexts = indexMergerRequest.getRepositoriesIds().stream().map(id -> repositoryRegistry.getRepository(id).getIndexingContext()).collect(Collectors.toList());
try {
ArchivaIndexingContext result = idxManager.mergeContexts(destinationRepository, sourceContexts, indexMergerRequest.isPackIndex());
if ( indexMergerRequest.isTemporary() )
{
String tempRepoId = destinationRepository.getId()+System.currentTimeMillis();
temporaryGroupIndexes.add( new TemporaryGroupIndex( mergedIndexDirectory, tempRepoId, groupId,
indexMergerRequest.getMergedIndexTtl() ) );
temporaryContextes.add(result);
}
return result;
} catch (IndexCreationFailedException e) {
throw new IndexMergerException("Index merging failed " + e.getMessage(), e);
}
if ( indexMergerRequest.isTemporary() )
{
temporaryGroupIndexes.add( new TemporaryGroupIndex( mergedIndexDirectory, tempRepoId, groupId,
indexMergerRequest.getMergedIndexTtl() ) );
}
} finally {
stopWatch.stop();
log.info( "merged index for repos {} in {} s", indexMergerRequest.getRepositoriesIds(),
stopWatch.getTime() );
return new MavenIndexContext(repositoryRegistry.getRepositoryGroup(groupId), mergedCtx);
}
catch ( IOException e)
{
throw new IndexMergerException( e.getMessage(), e );
}
finally
{
runningGroups.remove( groupId );
stopWatch.getTime() );
runningGroups.remove(groupId);
}
}
@ -127,11 +125,10 @@ public class BasicIndexMerger implements IndexMerger
try
{
Optional<IndexingContext> ctxOpt = temporaryContextes.stream( ).filter( ctx -> ctx.getId( ).equals( temporaryGroupIndex.getIndexId( ) ) ).findFirst( );
Optional<ArchivaIndexingContext> ctxOpt = temporaryContextes.stream( ).filter( ctx -> ctx.getId( ).equals( temporaryGroupIndex.getIndexId( ) ) ).findFirst( );
if (ctxOpt.isPresent()) {
IndexingContext ctx = ctxOpt.get();
indexer.closeIndexingContext( ctx, true );
ArchivaIndexingContext ctx = ctxOpt.get();
ctx.close(true);
temporaryGroupIndexes.remove( temporaryGroupIndex );
temporaryContextes.remove( ctx );
Path directory = temporaryGroupIndex.getDirectory();
@ -152,5 +149,4 @@ public class BasicIndexMerger implements IndexMerger
{
return this.temporaryGroupIndexes;
}
}

231
archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/RepositoryRegistry.java
View File

@ -56,7 +56,7 @@ import java.util.stream.Stream;
* The modification methods addXX and removeXX persist the changes immediately to the configuration. If the
* configuration save fails the changes are rolled back.
*
* TODO: Audit events should be sent, but we don't want dependency to the repsitory-metadata-api
* TODO: Audit events
*/
@Service( "repositoryRegistry" )
public class RepositoryRegistry implements ConfigurationListener, RepositoryEventHandler, RepositoryEventListener {
@ -241,7 +241,7 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven
}
private ArchivaIndexManager getIndexManager(RepositoryType type) {
public ArchivaIndexManager getIndexManager(RepositoryType type) {
return indexManagerFactory.getIndexManager(type);
}
@ -478,10 +478,14 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven
log.debug("Managed repo");
return managedRepositories.get( repoId );
}
else
else if (remoteRepositories.containsKey(repoId))
{
log.debug("Remote repo");
return remoteRepositories.get( repoId );
} else if (repositoryGroups.containsKey(repoId)) {
return repositoryGroups.get(repoId);
} else {
return null;
}
}
finally
@ -676,6 +680,153 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven
}
}
/**
* Adds a new repository group to the current list, or replaces the repository group definition with
* the same id, if it exists already.
* The change is saved to the configuration immediately.
*
* @param repositoryGroup the new repository group.
* @throws RepositoryException if the new repository group could not be saved to the configuration.
*/
public RepositoryGroup putRepositoryGroup( RepositoryGroup repositoryGroup ) throws RepositoryException
{
rwLock.writeLock( ).lock( );
try
{
final String id = repositoryGroup.getId();
RepositoryGroup originRepo = repositoryGroups.put( id, repositoryGroup );
try
{
if (originRepo!=null) {
originRepo.close();
}
RepositoryProvider provider = getProvider( repositoryGroup.getType() );
RepositoryGroupConfiguration newCfg = provider.getRepositoryGroupConfiguration( repositoryGroup );
Configuration configuration = getArchivaConfiguration( ).getConfiguration( );
updateRepositoryReferences( provider, repositoryGroup, newCfg );
RepositoryGroupConfiguration oldCfg = configuration.findRepositoryGroupById( id );
if (oldCfg!=null) {
configuration.removeRepositoryGroup( oldCfg );
}
configuration.addRepositoryGroup( newCfg );
getArchivaConfiguration( ).save( configuration );
return repositoryGroup;
}
catch ( Exception e )
{
// Rollback
if ( originRepo != null )
{
repositoryGroups.put( id, originRepo );
} else {
repositoryGroups.remove(id);
}
log.error("Exception during configuration update {}", e.getMessage(), e);
throw new RepositoryException( "Could not save the configuration" + (e.getMessage( )==null?"":": "+e.getMessage()) );
}
}
finally
{
rwLock.writeLock( ).unlock( );
}
}
/**
* Adds a new repository group or updates the repository with the same id, if it exists already.
* The configuration is saved immediately.
*
* @param repositoryGroupConfiguration the repository configuration
* @return the updated or created repository
* @throws RepositoryException if an error occurs, or the configuration is not valid.
*/
public RepositoryGroup putRepositoryGroup( RepositoryGroupConfiguration repositoryGroupConfiguration) throws RepositoryException
{
rwLock.writeLock( ).lock( );
try
{
final String id = repositoryGroupConfiguration.getId();
final RepositoryType repositoryType = RepositoryType.valueOf( repositoryGroupConfiguration.getType() );
Configuration configuration = getArchivaConfiguration().getConfiguration();
RepositoryGroup repo = repositoryGroups.get(id);
RepositoryGroupConfiguration oldCfg = repo!=null ? getProvider( repositoryType ).getRepositoryGroupConfiguration( repo ) : null;
repo = putRepositoryGroup( repositoryGroupConfiguration, configuration );
try
{
getArchivaConfiguration().save(configuration);
}
catch ( IndeterminateConfigurationException | RegistryException e )
{
if (oldCfg!=null) {
getProvider( repositoryType ).updateRepositoryGroupInstance( (EditableRepositoryGroup) repo, oldCfg );
}
log.error("Could not save the configuration for repository group {}: {}", id, e.getMessage(),e );
throw new RepositoryException( "Could not save the configuration for repository group "+id+": "+e.getMessage() );
}
return repo;
}
finally
{
rwLock.writeLock( ).unlock( );
}
}
/**
* Adds a new repository group or updates the repository group with the same id. The given configuration object is updated, but
* the configuration is not saved.
*
* @param repositoryGroupConfiguration the new or changed repository configuration
* @param configuration the configuration object
* @return the new or updated repository
* @throws RepositoryException if the configuration cannot be saved or updated
*/
@SuppressWarnings( "unchecked" )
public RepositoryGroup putRepositoryGroup( RepositoryGroupConfiguration repositoryGroupConfiguration, Configuration configuration) throws RepositoryException
{
rwLock.writeLock( ).lock( );
try
{
final String id = repositoryGroupConfiguration.getId();
final RepositoryType repoType = RepositoryType.valueOf( repositoryGroupConfiguration.getType() );
RepositoryGroup repo;
setRepositoryGroupDefaults(repositoryGroupConfiguration);
if (repositoryGroups.containsKey( id )) {
repo = repositoryGroups.get(id);
if (repo instanceof EditableRepositoryGroup)
{
getProvider( repoType ).updateRepositoryGroupInstance( (EditableRepositoryGroup) repo, repositoryGroupConfiguration );
} else {
throw new RepositoryException( "The repository is not editable "+id );
}
} else
{
repo = getProvider( repoType ).createRepositoryGroup( repositoryGroupConfiguration );
repo.addListener(this);
repositoryGroups.put(id, repo);
}
updateRepositoryReferences( getProvider( repoType ), repo, repositoryGroupConfiguration );
replaceOrAddRepositoryConfig( repositoryGroupConfiguration, configuration );
return repo;
}
finally
{
rwLock.writeLock( ).unlock( );
}
}
private void setRepositoryGroupDefaults(RepositoryGroupConfiguration repositoryGroupConfiguration) {
if (StringUtils.isEmpty(repositoryGroupConfiguration.getMergedIndexPath())) {
repositoryGroupConfiguration.setMergedIndexPath(".indexer");
}
if (repositoryGroupConfiguration.getMergedIndexTtl()<=0) {
repositoryGroupConfiguration.setMergedIndexTtl(300);
}
if (StringUtils.isEmpty(repositoryGroupConfiguration.getCronExpression())) {
repositoryGroupConfiguration.setCronExpression("0 0 03 ? * MON");
}
}
private void replaceOrAddRepositoryConfig(ManagedRepositoryConfiguration managedRepositoryConfiguration, Configuration configuration) {
ManagedRepositoryConfiguration oldCfg = configuration.findManagedRepositoryById( managedRepositoryConfiguration.getId() );
if ( oldCfg !=null) {
@ -692,6 +843,14 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven
configuration.addRemoteRepository( remoteRepositoryConfiguration );
}
private void replaceOrAddRepositoryConfig(RepositoryGroupConfiguration repositoryGroupConfiguration, Configuration configuration) {
RepositoryGroupConfiguration oldCfg = configuration.findRepositoryGroupById( repositoryGroupConfiguration.getId() );
if ( oldCfg !=null) {
configuration.removeRepositoryGroup( oldCfg );
}
configuration.addRepositoryGroup( repositoryGroupConfiguration);
}
public RemoteRepository putRepository( RemoteRepository remoteRepository, Configuration configuration) throws RepositoryException
{
rwLock.writeLock( ).lock( );
@ -874,7 +1033,9 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven
removeRepository( (RemoteRepository)repo );
} else if (repo instanceof ManagedRepository) {
removeRepository( (ManagedRepository)repo);
} else {
} else if (repo instanceof RepositoryGroup ) {
removeRepositoryGroup((RepositoryGroup) repo);
}else {
throw new RepositoryException( "Repository type not known: "+repo.getClass() );
}
}
@ -942,6 +1103,68 @@ public class RepositoryRegistry implements ConfigurationListener, RepositoryEven
}
/**
* Removes a repository group from the registry and configuration, if it exists.
* The change is saved to the configuration immediately.
*
* @param repositoryGroup the repository group to remove
* @throws RepositoryException if a error occurs during configuration save
*/
public void removeRepositoryGroup( RepositoryGroup repositoryGroup ) throws RepositoryException
{
final String id = repositoryGroup.getId();
RepositoryGroup repo = getRepositoryGroup( id );
if (repo!=null) {
rwLock.writeLock().lock();
try {
repo = repositoryGroups.remove( id );
if (repo!=null) {
repo.close();
Configuration configuration = getArchivaConfiguration().getConfiguration();
RepositoryGroupConfiguration cfg = configuration.findRepositoryGroupById( id );
if (cfg!=null) {
configuration.removeRepositoryGroup( cfg );
}
getArchivaConfiguration().save( configuration );
}
}
catch ( RegistryException | IndeterminateConfigurationException e )
{
// Rollback
log.error("Could not save config after repository removal: {}", e.getMessage(), e);
repositoryGroups.put(repo.getId(), repo);
throw new RepositoryException( "Could not save configuration after repository removal: "+e.getMessage() );
} finally
{
rwLock.writeLock().unlock();
}
}
}
public void removeRepositoryGroup(RepositoryGroup repositoryGroup, Configuration configuration) throws RepositoryException
{
final String id = repositoryGroup.getId();
RepositoryGroup repo = getRepositoryGroup( id );
if (repo!=null) {
rwLock.writeLock().lock();
try {
repo = repositoryGroups.remove( id );
if (repo!=null) {
repo.close();
RepositoryGroupConfiguration cfg = configuration.findRepositoryGroupById( id );
if (cfg!=null) {
configuration.removeRepositoryGroup( cfg );
}
}
} finally
{
rwLock.writeLock().unlock();
}
}
}
private void doRemoveRepo(RemoteRepository repo, Configuration configuration) {
repo.close();
RemoteRepositoryConfiguration cfg = configuration.findRemoteRepositoryById(repo.getId());

8
archiva-modules/archiva-base/archiva-repository-layer/src/main/resources/META-INF/spring-context.xml
View File

@ -28,6 +28,12 @@
default-lazy-init="true">
<context:annotation-config/>
<context:component-scan base-package="org.apache.archiva.repository,org.apache.archiva.repository.content"/>
<context:component-scan base-package="org.apache.archiva.repository,org.apache.archiva.repository.content,org.apache.archiva.indexer.merger"/>
<bean name="taskScheduler#mergeRemoteIndexes"
class="org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler">
<property name="poolSize" value="4"/>
<property name="threadGroupName" value="mergeRemoteIndexes"/>
</bean>
</beans>

6
archiva-modules/archiva-base/archiva-repository-layer/src/test/java/org/apache/archiva/repository/mock/ArchivaIndexManagerMock.java
View File

@ -29,6 +29,7 @@ import org.springframework.stereotype.Service;
import java.net.URI;
import java.util.Collection;
import java.util.List;
/**
* @author Martin Stockhammer <martin_s@apache.org>
@ -87,4 +88,9 @@ public class ArchivaIndexManagerMock implements ArchivaIndexManager {
public void updateLocalIndexPath(Repository repo) {
}
@Override
public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts, boolean packIndex) throws UnsupportedOperationException, IndexCreationFailedException {
return null;
}
}

198
archiva-modules/archiva-maven/archiva-maven-indexer/src/main/java/org/apache/archiva/indexer/maven/DefaultIndexMerger.java
View File

@ -1,198 +0,0 @@
package org.apache.archiva.indexer.maven;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.archiva.common.utils.FileUtils;
import org.apache.archiva.indexer.ArchivaIndexingContext;
import org.apache.archiva.indexer.UnsupportedBaseContextException;
import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.merger.IndexMergerRequest;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.archiva.repository.RepositoryType;
import org.apache.commons.lang.time.StopWatch;
import org.apache.maven.index.Indexer;
import org.apache.maven.index.context.ContextMemberProvider;
import org.apache.maven.index.context.IndexCreator;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.StaticContextMemberProvider;
import org.apache.maven.index.packer.IndexPacker;
import org.apache.maven.index.packer.IndexPackingRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;
/**
* @author Olivier Lamy
* @since 1.4-M2
*/
@Service("indexMerger#default")
public class DefaultIndexMerger
implements IndexMerger
{
@Inject
RepositoryRegistry repositoryRegistry;
private Logger log = LoggerFactory.getLogger( getClass() );
private final IndexPacker indexPacker;
private Indexer indexer;
private final List<IndexCreator> indexCreators;
private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<>();
private List<IndexingContext> temporaryContextes = new CopyOnWriteArrayList<>( );
private List<String> runningGroups = new CopyOnWriteArrayList<>();
@Inject
public DefaultIndexMerger( Indexer indexer, IndexPacker indexPacker, List<IndexCreator> indexCreators )
{
this.indexer = indexer;
this.indexPacker = indexPacker;
this.indexCreators = indexCreators;
}
@Override
public ArchivaIndexingContext buildMergedIndex(IndexMergerRequest indexMergerRequest )
throws IndexMergerException
{
String groupId = indexMergerRequest.getGroupId();
if ( runningGroups.contains( groupId ) )
{
log.info( "skip build merge remote indexes for id: '{}' as already running", groupId );
return null;
}
runningGroups.add( groupId );
StopWatch stopWatch = new StopWatch();
stopWatch.reset();
stopWatch.start();
Path mergedIndexDirectory = indexMergerRequest.getMergedIndexDirectory();
String tempRepoId = mergedIndexDirectory.getFileName().toString();
try
{
Path indexLocation = mergedIndexDirectory.resolve( indexMergerRequest.getMergedIndexPath() );
List<IndexingContext> members = indexMergerRequest.getRepositoriesIds( ).stream( ).map( id ->
repositoryRegistry.getRepository( id ) ).filter( repo -> repo.getType().equals( RepositoryType.MAVEN ) )
.map( repo -> {
try
{
return repo.getIndexingContext().getBaseContext( IndexingContext.class );
}
catch ( UnsupportedBaseContextException e )
{
return null;
// Ignore
}
} ).filter( Objects::nonNull ).collect( Collectors.toList() );
ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
indexLocation.toFile(), true, memberProvider);
mergedCtx.optimize();
if ( indexMergerRequest.isPackIndex() )
{
IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
mergedCtx.acquireIndexSearcher().getIndexReader(), //
indexLocation.toFile() );
indexPacker.packIndex( request );
}
if ( indexMergerRequest.isTemporary() )
{
temporaryGroupIndexes.add( new TemporaryGroupIndex( mergedIndexDirectory, tempRepoId, groupId,
indexMergerRequest.getMergedIndexTtl() ) );
temporaryContextes.add(mergedCtx);
}
stopWatch.stop();
log.info( "merged index for repos {} in {} s", indexMergerRequest.getRepositoriesIds(),
stopWatch.getTime() );
return new MavenIndexContext(repositoryRegistry.getRepositoryGroup(groupId), mergedCtx);
}
catch ( IOException e)
{
throw new IndexMergerException( e.getMessage(), e );
}
finally
{
runningGroups.remove( groupId );
}
}
@Async
@Override
public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex )
{
if ( temporaryGroupIndex == null )
{
return;
}
try
{
Optional<IndexingContext> ctxOpt = temporaryContextes.stream( ).filter( ctx -> ctx.getId( ).equals( temporaryGroupIndex.getIndexId( ) ) ).findFirst( );
if (ctxOpt.isPresent()) {
IndexingContext ctx = ctxOpt.get();
indexer.closeIndexingContext( ctx, true );
temporaryGroupIndexes.remove( temporaryGroupIndex );
temporaryContextes.remove( ctx );
Path directory = temporaryGroupIndex.getDirectory();
if ( directory != null && Files.exists(directory) )
{
FileUtils.deleteDirectory( directory );
}
}
}
catch ( IOException e )
{
log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e );
}
}
@Override
public Collection<TemporaryGroupIndex> getTemporaryGroupIndexes()
{
return this.temporaryGroupIndexes;
}
}

61
archiva-modules/archiva-maven/archiva-maven-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java
View File

@ -28,6 +28,8 @@ import org.apache.archiva.indexer.ArchivaIndexingContext;
import org.apache.archiva.indexer.IndexCreationFailedException;
import org.apache.archiva.indexer.IndexUpdateFailedException;
import org.apache.archiva.indexer.UnsupportedBaseContextException;
import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.apache.archiva.proxy.ProxyRegistry;
import org.apache.archiva.proxy.maven.WagonFactory;
import org.apache.archiva.proxy.maven.WagonFactoryException;
@ -53,8 +55,10 @@ import org.apache.maven.index.IndexerEngine;
import org.apache.maven.index.Scanner;
import org.apache.maven.index.ScanningRequest;
import org.apache.maven.index.ScanningResult;
import org.apache.maven.index.context.ContextMemberProvider;
import org.apache.maven.index.context.IndexCreator;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.StaticContextMemberProvider;
import org.apache.maven.index.packer.IndexPacker;
import org.apache.maven.index.packer.IndexPackingRequest;
import org.apache.maven.index.updater.IndexUpdateRequest;
@ -91,6 +95,7 @@ import java.nio.file.Paths;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.stream.Collectors;
@ -535,6 +540,62 @@ public class MavenIndexManager implements ArchivaIndexManager {
}
}
@Override
public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts,
boolean packIndex) throws UnsupportedOperationException,
IndexCreationFailedException, IllegalArgumentException {
if (!destinationRepo.supportsFeature(IndexCreationFeature.class)) {
throw new IllegalArgumentException("The given repository does not support the indexcreation feature");
}
Path mergedIndexDirectory = null;
try {
mergedIndexDirectory = Files.createTempDirectory("archivaMergedIndex");
} catch (IOException e) {
log.error("Could not create temporary directory for merged index: {}", e.getMessage(), e);
throw new IndexCreationFailedException("IO error while creating temporary directory for merged index: "+e.getMessage(), e);
}
IndexCreationFeature indexCreationFeature = destinationRepo.getFeature(IndexCreationFeature.class).get();
if (indexCreationFeature.getLocalIndexPath()== null) {
throw new IllegalArgumentException("The given repository does not have a local index path");
}
StorageAsset destinationPath = indexCreationFeature.getLocalIndexPath();
String tempRepoId = mergedIndexDirectory.getFileName().toString();
try
{
Path indexLocation = destinationPath.getFilePath();
List<IndexingContext> members = contexts.stream( ).filter(ctx -> ctx.supports(IndexingContext.class)).map( ctx ->
{
try {
return ctx.getBaseContext(IndexingContext.class);
} catch (UnsupportedBaseContextException e) {
// does not happen here
return null;
}
}).filter( Objects::nonNull ).collect( Collectors.toList() );
ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
indexLocation.toFile(), true, memberProvider);
mergedCtx.optimize();
if ( packIndex )
{
IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
mergedCtx.acquireIndexSearcher().getIndexReader(), //
indexLocation.toFile() );
indexPacker.packIndex( request );
}
return new MavenIndexContext(destinationRepo, mergedCtx);
}
catch ( IOException e)
{
throw new IndexCreationFailedException( "IO Error during index merge: "+ e.getMessage(), e );
}
}
private StorageAsset getIndexPath(URI indexDir, Path repoDir, String defaultDir) throws IOException
{
String indexPath = indexDir.getPath();

6
archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/repository/mock/ArchivaIndexManagerMock.java
View File

@ -29,6 +29,7 @@ import org.springframework.stereotype.Service;
import java.net.URI;
import java.util.Collection;
import java.util.List;
/**
* @author Martin Stockhammer <martin_s@apache.org>
@ -87,4 +88,9 @@ public class ArchivaIndexManagerMock implements ArchivaIndexManager {
public void updateLocalIndexPath(Repository repo) {
}
@Override
public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts, boolean packIndex) throws UnsupportedOperationException, IndexCreationFailedException {
return null;
}
}

19
archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven2/MavenRepositoryProvider.java
View File

@ -257,20 +257,13 @@ public class MavenRepositoryProvider implements RepositoryProvider {
repositoryGroup.setSchedulingDefinition(configuration.getCronExpression());
if (repositoryGroup.supportsFeature( IndexCreationFeature.class )) {
IndexCreationFeature indexCreationFeature = repositoryGroup.getFeature( IndexCreationFeature.class ).get();
try
indexCreationFeature.setIndexPath( getURIFromString(configuration.getMergedIndexPath()) );
Path localPath = Paths.get(configuration.getMergedIndexPath());
if (localPath.isAbsolute()) {
indexCreationFeature.setLocalIndexPath( new FilesystemAsset(localPath.getFileName().toString(), localPath) );
} else
{
indexCreationFeature.setIndexPath( new URI(configuration.getMergedIndexPath()) );
Path localPath = Paths.get(indexCreationFeature.getIndexPath());
if (localPath.isAbsolute()) {
indexCreationFeature.setLocalIndexPath( new FilesystemAsset(localPath.getFileName().toString(), localPath) );
} else
{
indexCreationFeature.setLocalIndexPath( new FilesystemAsset(localPath.toString(), archivaConfiguration.getRepositoryGroupBaseDir( ).resolve( localPath )));
}
}
catch ( URISyntaxException e )
{
log.error("Could not set the index path for repository group {}", repositoryGroup.getId());
indexCreationFeature.setLocalIndexPath( new FilesystemAsset(localPath.toString(), archivaConfiguration.getRepositoryGroupBaseDir( ).resolve( localPath )));
}
}
// References to other repositories are set filled by the registry

5
archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/index/mock/ArchivaIndexManagerMock.java
View File

@ -521,6 +521,11 @@ public class ArchivaIndexManagerMock implements ArchivaIndexManager {
}
}
@Override
public ArchivaIndexingContext mergeContexts(Repository destinationRepo, List<ArchivaIndexingContext> contexts, boolean packIndex) throws UnsupportedOperationException, IndexCreationFailedException {
return null;
}
private StorageAsset getIndexPath( Repository repo) throws IOException {
IndexCreationFeature icf = repo.getFeature(IndexCreationFeature.class).get();

6
archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven2/MavenRepositoryProviderTest.java
View File

@ -355,7 +355,11 @@ public class MavenRepositoryProviderTest
assertEquals("Group 2", grp.getName());
assertEquals("0 0 03 ? * MON", grp.getSchedulingDefinition());
IndexCreationFeature indexCreationFeature = grp.getFeature( IndexCreationFeature.class ).get();
assertEquals(".index-abc", indexCreationFeature.getIndexPath());
try {
assertEquals(new URI("file://.index-abc"), indexCreationFeature.getIndexPath());
} catch (URISyntaxException e) {
e.printStackTrace();
}
assertEquals(504, grp.getMergedIndexTTL());
assertEquals(0, grp.getRepositories().size());
// assertTrue(grp.getRepositories().stream().anyMatch(r -> "test01".equals(r.getId())));