Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

start refactoring merging remote indexes to have a cron job doing it 

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1551122 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Olivier Lamy 2013-12-16 06:53:27 +00:00
parent c2d4f8bce1
commit 28898793f5
7 changed files with 145 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo
View File

@ -856,6 +856,16 @@
<defaultValue>30</defaultValue> <defaultValue>30</defaultValue>
<description>The time to live of the merged index of the repository group.</description> <description>The time to live of the merged index of the repository group.</description>
</field> </field>
<field>
<name>mergedIndexCronExpression</name>
<version>1.0.0+</version>
<type>String</type>
<description>
When to run the index merging for this group.
No default value.
</description>
<defaultValue></defaultValue>
</field>
<field> <field>
<name>repositories</name> <name>repositories</name>
<version>1.2.0+</version> <version>1.2.0+</version>

4
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java
View File

@ -80,8 +80,8 @@ public class DefaultIndexMerger
StopWatch stopWatch = new StopWatch(); StopWatch stopWatch = new StopWatch();
stopWatch.reset(); stopWatch.reset();
stopWatch.start(); stopWatch.start();
File tempRepoFile = Files.createTempDir();
tempRepoFile.deleteOnExit(); File tempRepoFile = indexMergerRequest.getMergedIndexDirectory();
String tempRepoId = tempRepoFile.getName(); String tempRepoId = tempRepoFile.getName();

33
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java
View File

@ -18,6 +18,7 @@ package org.apache.archiva.indexer.merger;
* under the License. * under the License.
*/ */
import java.io.File;
import java.util.Collection; import java.util.Collection;
/** /**
@ -44,6 +45,8 @@ public class IndexMergerRequest
private int mergedIndexTtl; private int mergedIndexTtl;
private File mergedIndexDirectory;
public IndexMergerRequest( Collection<String> repositoriesIds, boolean packIndex, String groupId ) public IndexMergerRequest( Collection<String> repositoriesIds, boolean packIndex, String groupId )
{ {
this.repositoriesIds = repositoriesIds; this.repositoriesIds = repositoriesIds;
@ -54,8 +57,8 @@ public class IndexMergerRequest
/** /**
* @since 1.4-M4 * @since 1.4-M4
*/ */
public IndexMergerRequest(Collection<String> repositoriesIds, boolean packIndex, String groupId, public IndexMergerRequest( Collection<String> repositoriesIds, boolean packIndex, String groupId,
String mergedIndexPath, int mergedIndexTtl) String mergedIndexPath, int mergedIndexTtl )
{ {
this.repositoriesIds = repositoriesIds; this.repositoriesIds = repositoriesIds;
this.packIndex = packIndex; this.packIndex = packIndex;
@ -104,14 +107,33 @@ public class IndexMergerRequest
this.mergedIndexPath = mergedIndexPath; this.mergedIndexPath = mergedIndexPath;
} }
public int getMergedIndexTtl() { public int getMergedIndexTtl()
{
return mergedIndexTtl; return mergedIndexTtl;
} }
public void setMergedIndexTtl(int mergedIndexTtl) { public void setMergedIndexTtl( int mergedIndexTtl )
{
this.mergedIndexTtl = mergedIndexTtl; this.mergedIndexTtl = mergedIndexTtl;
} }
public File getMergedIndexDirectory()
{
return mergedIndexDirectory;
}
public void setMergedIndexDirectory( File mergedIndexDirectory )
{
this.mergedIndexDirectory = mergedIndexDirectory;
}
public IndexMergerRequest mergedIndexDirectory( File mergedIndexDirectory )
{
this.mergedIndexDirectory = mergedIndexDirectory;
return this;
}
@Override @Override
public String toString() public String toString()
{ {
@ -120,7 +142,8 @@ public class IndexMergerRequest
sb.append( ", packIndex=" ).append( packIndex ); sb.append( ", packIndex=" ).append( packIndex );
sb.append( ", groupId='" ).append( groupId ).append( '\'' ); sb.append( ", groupId='" ).append( groupId ).append( '\'' );
sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' ); sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' );
sb.append( ", mergedIndexTtl='" ).append( mergedIndexTtl ).append( '\'' ); sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl );
sb.append( ", mergedIndexDirectory='" ).append( mergedIndexDirectory ).append( '\'' );
sb.append( '}' ); sb.append( '}' );
return sb.toString(); return sb.toString();
} }

30
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RepositoryGroup.java
View File

@ -51,6 +51,12 @@ public class RepositoryGroup
*/ */
private int mergedIndexTtl = 30; private int mergedIndexTtl = 30;
/**
* default model value is empty so none
* @since 2.0.0
*/
private String mergedIndexCronExpression;
public RepositoryGroup() public RepositoryGroup()
{ {
// no op // no op
@ -160,6 +166,22 @@ public class RepositoryGroup
return this; return this;
} }
public String getMergedIndexCronExpression()
{
return mergedIndexCronExpression;
}
public void setMergedIndexCronExpression( String mergedIndexCronExpression )
{
this.mergedIndexCronExpression = mergedIndexCronExpression;
}
public RepositoryGroup mergedIndexCronExpression( String mergedIndexCronExpression )
{
this.mergedIndexCronExpression = mergedIndexCronExpression;
return this;
}
public boolean equals( Object other ) public boolean equals( Object other )
{ {
if ( this == other ) if ( this == other )
@ -188,10 +210,12 @@ public class RepositoryGroup
@Override @Override
public String toString() public String toString()
{ {
final StringBuilder sb = new StringBuilder(); final StringBuilder sb = new StringBuilder( "RepositoryGroup{" );
sb.append( "RepositoryGroup" ); sb.append( "id='" ).append( id ).append( '\'' );
sb.append( "{id='" ).append( id ).append( '\'' );
sb.append( ", repositories=" ).append( repositories ); sb.append( ", repositories=" ).append( repositories );
sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' );
sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl );
sb.append( ", mergedIndexCronExpression='" ).append( mergedIndexCronExpression ).append( '\'' );
sb.append( '}' ); sb.append( '}' );
return sb.toString(); return sb.toString();
} }

3
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/group/RepositoryGroupAdmin.java
View File

@ -22,6 +22,7 @@ import org.apache.archiva.admin.model.AuditInformation;
import org.apache.archiva.admin.model.RepositoryAdminException; import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.RepositoryGroup; import org.apache.archiva.admin.model.beans.RepositoryGroup;
import java.io.File;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -73,4 +74,6 @@ public interface RepositoryGroupAdmin
*/ */
Map<String, List<String>> getRepositoryToGroupMap() Map<String, List<String>> getRepositoryToGroupMap()
throws RepositoryAdminException; throws RepositoryAdminException;
File getMergedIndexDirectory( String repositoryGroupId );
} }

26
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java
View File

@ -33,7 +33,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.inject.Inject; import javax.inject.Inject;
import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
@ -58,6 +60,25 @@ public class DefaultRepositoryGroupAdmin
@Inject @Inject
private ManagedRepositoryAdmin managedRepositoryAdmin; private ManagedRepositoryAdmin managedRepositoryAdmin;
private File groupsDirectory;
@PostConstruct
public void initialize()
{
String appServerBase = getRegistry().getString( "appserver.base" );
groupsDirectory = new File( appServerBase + File.separatorChar + "groups" );
if ( !groupsDirectory.exists() )
{
groupsDirectory.mkdirs();
}
}
@Override
public File getMergedIndexDirectory( String repositoryGroupId )
{
return new File( groupsDirectory, repositoryGroupId );
}
public List<RepositoryGroup> getRepositoriesGroups() public List<RepositoryGroup> getRepositoriesGroups()
throws RepositoryAdminException throws RepositoryAdminException
{ {
@ -68,7 +89,8 @@ public class DefaultRepositoryGroupAdmin
{ {
repositoriesGroups.add( new RepositoryGroup( repositoryGroupConfiguration.getId(), new ArrayList<String>( repositoriesGroups.add( new RepositoryGroup( repositoryGroupConfiguration.getId(), new ArrayList<String>(
repositoryGroupConfiguration.getRepositories() ) ).mergedIndexPath( repositoryGroupConfiguration.getRepositories() ) ).mergedIndexPath(
repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl( repositoryGroupConfiguration.getMergedIndexTtl() ) ); repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl(
repositoryGroupConfiguration.getMergedIndexTtl() ) );
} }
return repositoriesGroups; return repositoriesGroups;
@ -285,7 +307,7 @@ public class DefaultRepositoryGroupAdmin
"Invalid character(s) found in identifier. Only the following characters are allowed: alphanumeric, '.', '-' and '_'" ); "Invalid character(s) found in identifier. Only the following characters are allowed: alphanumeric, '.', '-' and '_'" );
} }
if ( repositoryGroup.getMergedIndexTtl() <= 0) if ( repositoryGroup.getMergedIndexTtl() <= 0 )
{ {
throw new RepositoryAdminException( "Merged Index TTL must be greater than 0." ); throw new RepositoryAdminException( "Merged Index TTL must be greater than 0." );
} }

75
archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java
View File

@ -19,6 +19,7 @@ package org.apache.archiva.webdav;
* under the License. * under the License.
*/ */
import com.google.common.io.Files;
import org.apache.archiva.admin.model.RepositoryAdminException; import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.ManagedRepository; import org.apache.archiva.admin.model.beans.ManagedRepository;
import org.apache.archiva.admin.model.beans.RemoteRepository; import org.apache.archiva.admin.model.beans.RemoteRepository;
@ -37,6 +38,7 @@ import org.apache.archiva.configuration.RepositoryGroupConfiguration;
import org.apache.archiva.indexer.merger.IndexMerger; import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.IndexMergerException; import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.merger.IndexMergerRequest; import org.apache.archiva.indexer.merger.IndexMergerRequest;
import org.apache.archiva.indexer.merger.MergedRemoteIndexesTaskJob;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex; import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.apache.archiva.indexer.search.RepositorySearch; import org.apache.archiva.indexer.search.RepositorySearch;
import org.apache.archiva.maven2.metadata.MavenMetadataReader; import org.apache.archiva.maven2.metadata.MavenMetadataReader;
@ -115,7 +117,7 @@ import java.util.Set;
/** /**
* *
*/ */
@Service( "davResourceFactory#archiva" ) @Service("davResourceFactory#archiva")
public class ArchivaDavResourceFactory public class ArchivaDavResourceFactory
implements DavResourceFactory, Auditable implements DavResourceFactory, Auditable
{ {
@ -134,7 +136,7 @@ public class ArchivaDavResourceFactory
private RepositoryRequest repositoryRequest; private RepositoryRequest repositoryRequest;
@Inject @Inject
@Named( value = "repositoryProxyConnectors#default" ) @Named(value = "repositoryProxyConnectors#default")
private RepositoryProxyConnectors connectors; private RepositoryProxyConnectors connectors;
@Inject @Inject
@ -149,7 +151,7 @@ public class ArchivaDavResourceFactory
private ServletAuthenticator servletAuth; private ServletAuthenticator servletAuth;
@Inject @Inject
@Named( value = "httpAuthenticator#basic" ) @Named(value = "httpAuthenticator#basic")
private HttpAuthenticator httpAuth; private HttpAuthenticator httpAuth;
@Inject @Inject
@ -176,11 +178,11 @@ public class ArchivaDavResourceFactory
private Digester digestMd5; private Digester digestMd5;
@Inject @Inject
@Named( value = "archivaTaskScheduler#repository" ) @Named(value = "archivaTaskScheduler#repository")
private RepositoryArchivaTaskScheduler scheduler; private RepositoryArchivaTaskScheduler scheduler;
@Inject @Inject
@Named(value= "fileLockManager#default") @Named(value = "fileLockManager#default")
private FileLockManager fileLockManager; private FileLockManager fileLockManager;
private ApplicationContext applicationContext; private ApplicationContext applicationContext;
@ -345,12 +347,13 @@ public class ArchivaDavResourceFactory
if ( metadataChecksum.exists() ) if ( metadataChecksum.exists() )
{ {
LogicalResource logicalResource = LogicalResource logicalResource =
new LogicalResource (getLogicalResource( archivaLocator, null, false ) ); new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
resource = resource =
new ArchivaDavResource( metadataChecksum.getAbsolutePath(), logicalResource.getPath(), null, new ArchivaDavResource( metadataChecksum.getAbsolutePath(), logicalResource.getPath(), null,
request.getRemoteAddr(), activePrincipal, request.getDavSession(), request.getRemoteAddr(), activePrincipal, request.getDavSession(),
archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager ); archivaLocator, this, mimeTypes, auditListeners, scheduler,
fileLockManager );
} }
} }
else else
@ -383,7 +386,8 @@ public class ArchivaDavResourceFactory
{ {
File resourceFile = writeMergedMetadataToFile( mergedMetadata, filePath ); File resourceFile = writeMergedMetadataToFile( mergedMetadata, filePath );
LogicalResource logicalResource = new LogicalResource( getLogicalResource( archivaLocator, null, false ) ); LogicalResource logicalResource =
new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
resource = resource =
new ArchivaDavResource( resourceFile.getAbsolutePath(), logicalResource.getPath(), null, new ArchivaDavResource( resourceFile.getAbsolutePath(), logicalResource.getPath(), null,
@ -404,7 +408,8 @@ public class ArchivaDavResourceFactory
catch ( DigesterException de ) catch ( DigesterException de )
{ {
throw new DavException( HttpServletResponse.SC_INTERNAL_SERVER_ERROR, throw new DavException( HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
"Error occurred while generating checksum files." + de.getMessage() ); "Error occurred while generating checksum files."
+ de.getMessage() );
} }
} }
} }
@ -445,7 +450,8 @@ public class ArchivaDavResourceFactory
File resourceFile = new File( temporaryIndexDirectory, requestedFileName ); File resourceFile = new File( temporaryIndexDirectory, requestedFileName );
resource = new ArchivaDavResource( resourceFile.getAbsolutePath(), requestedFileName, null, resource = new ArchivaDavResource( resourceFile.getAbsolutePath(), requestedFileName, null,
request.getRemoteAddr(), activePrincipal, request.getDavSession(), request.getRemoteAddr(), activePrincipal, request.getDavSession(),
archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager ); archivaLocator, this, mimeTypes, auditListeners, scheduler,
fileLockManager );
} }
else else
@ -523,32 +529,39 @@ public class ArchivaDavResourceFactory
{ {
// FIXME remove this hack // FIXME remove this hack
// but currently managedRepository can be null in case of group // but currently managedRepository can be null in case of group
String layout = managedRepository == null ? new ManagedRepository( ).getLayout() : managedRepository.getLayout(); String layout = managedRepository == null ? new ManagedRepository().getLayout() : managedRepository.getLayout();
RepositoryStorage repositoryStorage = RepositoryStorage repositoryStorage =
this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class ); this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class );
String path = repositoryStorage.getFilePath( String path = repositoryStorage.getFilePath(
useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(), managedRepository ); useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(),
managedRepository );
log.debug( "found path {} for resourcePath: '{}' with managedRepo '{}' and layout '{}'", path, log.debug( "found path {} for resourcePath: '{}' with managedRepo '{}' and layout '{}'", path,
archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(), layout ); archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(),
layout );
return path; return path;
} }
private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator, ManagedRepositoryContent managedRepositoryContent, String contextPath ) private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator,
ManagedRepositoryContent managedRepositoryContent, String contextPath )
throws DavException throws DavException
{ {
String layout = managedRepositoryContent.getRepository() == null ? new ManagedRepository( ).getLayout() : managedRepositoryContent.getRepository().getLayout(); String layout = managedRepositoryContent.getRepository() == null
? new ManagedRepository().getLayout()
: managedRepositoryContent.getRepository().getLayout();
RepositoryStorage repositoryStorage = RepositoryStorage repositoryStorage =
this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class ); this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class );
try try
{ {
return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(), managedRepositoryContent ); return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(),
managedRepositoryContent );
} }
catch ( RelocationException e ) catch ( RelocationException e )
{ {
String path = e.getPath(); String path = e.getPath();
log.debug( "Relocation to {}", path ); log.debug( "Relocation to {}", path );
throw new BrowserRedirectException(contextPath + ( StringUtils.startsWith( path, "/" ) ? "": "/" ) + path, e.getRelocationType() ); throw new BrowserRedirectException( contextPath + ( StringUtils.startsWith( path, "/" ) ? "" : "/" ) + path,
e.getRelocationType() );
} }
catch ( XMLException e ) catch ( XMLException e )
{ {
@ -566,7 +579,7 @@ public class ArchivaDavResourceFactory
if ( isAuthorized( request, managedRepositoryContent.getId() ) ) if ( isAuthorized( request, managedRepositoryContent.getId() ) )
{ {
// Maven Centric part ask evaluation if -SNAPSHOT // Maven Centric part ask evaluation if -SNAPSHOT
String path = evaluatePathWithVersion(archivaLocator, managedRepositoryContent, request.getContextPath()); String path = evaluatePathWithVersion( archivaLocator, managedRepositoryContent, request.getContextPath() );
if ( path.startsWith( "/" ) ) if ( path.startsWith( "/" ) )
{ {
path = path.substring( 1 ); path = path.substring( 1 );
@ -1272,12 +1285,14 @@ public class ArchivaDavResourceFactory
} }
Set<String> authzRepos = new HashSet<String>(); Set<String> authzRepos = new HashSet<String>();
String permission = WebdavMethodUtil.getMethodPermission( request.getMethod() );
for ( String repository : repositories ) for ( String repository : repositories )
{ {
try try
{ {
if ( servletAuth.isAuthorized( activePrincipal, repository, if ( servletAuth.isAuthorized( activePrincipal, repository, permission ) )
WebdavMethodUtil.getMethodPermission( request.getMethod() ) ) )
{ {
authzRepos.add( repository ); authzRepos.add( repository );
authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) ); authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) );
@ -1293,10 +1308,22 @@ public class ArchivaDavResourceFactory
} }
log.info( "generate temporary merged index for repository group '{}' for repositories '{}'", log.info( "generate temporary merged index for repository group '{}' for repositories '{}'",
repositoryGroupConfiguration.getId(), authzRepos ); repositoryGroupConfiguration.getId(), authzRepos );
IndexingContext indexingContext = indexMerger.buildMergedIndex(
new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(), File tempRepoFile = Files.createTempDir();
repositoryGroupConfiguration.getMergedIndexPath(), tempRepoFile.deleteOnExit();
repositoryGroupConfiguration.getMergedIndexTtl() ) );
IndexMergerRequest indexMergerRequest = new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(),
repositoryGroupConfiguration.getMergedIndexPath(),
repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory(
tempRepoFile );
MergedRemoteIndexesTaskJob job = new MergedRemoteIndexesTaskJob();
MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest taskRequest =
new MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest(indexMergerRequest, indexMerger);
IndexingContext indexingContext = job.execute( taskRequest ).getIndexingContext();
File mergedRepoDir = indexingContext.getIndexDirectoryFile(); File mergedRepoDir = indexingContext.getIndexDirectoryFile();
TemporaryGroupIndex temporaryGroupIndex = TemporaryGroupIndex temporaryGroupIndex =
new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId(), repositoryGroupConfiguration.getId(), new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId(), repositoryGroupConfiguration.getId(),