Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

start refactoring merging remote indexes to have a cron job doing it 

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1551122 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Olivier Lamy 2013-12-16 06:53:27 +00:00
parent c2d4f8bce1
commit 28898793f5
7 changed files with 145 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
archiva-modules/archiva-base/archiva-configuration/src/main/mdo/configuration.mdo
View File

@ -856,6 +856,16 @@
<defaultValue>30</defaultValue>
<description>The time to live of the merged index of the repository group.</description>
</field>
<field>
<name>mergedIndexCronExpression</name>
<version>1.0.0+</version>
<type>String</type>
<description>
When to run the index merging for this group.
No default value.
</description>
<defaultValue></defaultValue>
</field>
<field>
<name>repositories</name>
<version>1.2.0+</version>

4
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java
View File

@ -80,8 +80,8 @@ public class DefaultIndexMerger
StopWatch stopWatch = new StopWatch();
stopWatch.reset();
stopWatch.start();
File tempRepoFile = Files.createTempDir();
tempRepoFile.deleteOnExit();
File tempRepoFile = indexMergerRequest.getMergedIndexDirectory();
String tempRepoId = tempRepoFile.getName();

33
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerRequest.java
View File

@ -18,6 +18,7 @@ package org.apache.archiva.indexer.merger;
* under the License.
*/
import java.io.File;
import java.util.Collection;
/**
@ -44,6 +45,8 @@ public class IndexMergerRequest
private int mergedIndexTtl;
private File mergedIndexDirectory;
public IndexMergerRequest( Collection<String> repositoriesIds, boolean packIndex, String groupId )
{
this.repositoriesIds = repositoriesIds;
@ -54,8 +57,8 @@ public class IndexMergerRequest
/**
* @since 1.4-M4
*/
public IndexMergerRequest(Collection<String> repositoriesIds, boolean packIndex, String groupId,
String mergedIndexPath, int mergedIndexTtl)
public IndexMergerRequest( Collection<String> repositoriesIds, boolean packIndex, String groupId,
String mergedIndexPath, int mergedIndexTtl )
{
this.repositoriesIds = repositoriesIds;
this.packIndex = packIndex;
@ -104,14 +107,33 @@ public class IndexMergerRequest
this.mergedIndexPath = mergedIndexPath;
}
public int getMergedIndexTtl() {
public int getMergedIndexTtl()
{
return mergedIndexTtl;
}
public void setMergedIndexTtl(int mergedIndexTtl) {
public void setMergedIndexTtl( int mergedIndexTtl )
{
this.mergedIndexTtl = mergedIndexTtl;
}
public File getMergedIndexDirectory()
{
return mergedIndexDirectory;
}
public void setMergedIndexDirectory( File mergedIndexDirectory )
{
this.mergedIndexDirectory = mergedIndexDirectory;
}
public IndexMergerRequest mergedIndexDirectory( File mergedIndexDirectory )
{
this.mergedIndexDirectory = mergedIndexDirectory;
return this;
}
@Override
public String toString()
{
@ -120,7 +142,8 @@ public class IndexMergerRequest
sb.append( ", packIndex=" ).append( packIndex );
sb.append( ", groupId='" ).append( groupId ).append( '\'' );
sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' );
sb.append( ", mergedIndexTtl='" ).append( mergedIndexTtl ).append( '\'' );
sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl );
sb.append( ", mergedIndexDirectory='" ).append( mergedIndexDirectory ).append( '\'' );
sb.append( '}' );
return sb.toString();
}

30
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/beans/RepositoryGroup.java
View File

@ -51,6 +51,12 @@ public class RepositoryGroup
*/
private int mergedIndexTtl = 30;
/**
* default model value is empty so none
* @since 2.0.0
*/
private String mergedIndexCronExpression;
public RepositoryGroup()
{
// no op
@ -160,6 +166,22 @@ public class RepositoryGroup
return this;
}
public String getMergedIndexCronExpression()
{
return mergedIndexCronExpression;
}
public void setMergedIndexCronExpression( String mergedIndexCronExpression )
{
this.mergedIndexCronExpression = mergedIndexCronExpression;
}
public RepositoryGroup mergedIndexCronExpression( String mergedIndexCronExpression )
{
this.mergedIndexCronExpression = mergedIndexCronExpression;
return this;
}
public boolean equals( Object other )
{
if ( this == other )
@ -188,10 +210,12 @@ public class RepositoryGroup
@Override
public String toString()
{
final StringBuilder sb = new StringBuilder();
sb.append( "RepositoryGroup" );
sb.append( "{id='" ).append( id ).append( '\'' );
final StringBuilder sb = new StringBuilder( "RepositoryGroup{" );
sb.append( "id='" ).append( id ).append( '\'' );
sb.append( ", repositories=" ).append( repositories );
sb.append( ", mergedIndexPath='" ).append( mergedIndexPath ).append( '\'' );
sb.append( ", mergedIndexTtl=" ).append( mergedIndexTtl );
sb.append( ", mergedIndexCronExpression='" ).append( mergedIndexCronExpression ).append( '\'' );
sb.append( '}' );
return sb.toString();
}

3
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-api/src/main/java/org/apache/archiva/admin/model/group/RepositoryGroupAdmin.java
View File

@ -22,6 +22,7 @@ import org.apache.archiva.admin.model.AuditInformation;
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.RepositoryGroup;
import java.io.File;
import java.util.List;
import java.util.Map;
@ -73,4 +74,6 @@ public interface RepositoryGroupAdmin
*/
Map<String, List<String>> getRepositoryToGroupMap()
throws RepositoryAdminException;
File getMergedIndexDirectory( String repositoryGroupId );
}

26
archiva-modules/archiva-base/archiva-repository-admin/archiva-repository-admin-default/src/main/java/org/apache/archiva/admin/repository/group/DefaultRepositoryGroupAdmin.java
View File

@ -33,7 +33,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@ -58,6 +60,25 @@ public class DefaultRepositoryGroupAdmin
@Inject
private ManagedRepositoryAdmin managedRepositoryAdmin;
private File groupsDirectory;
@PostConstruct
public void initialize()
{
String appServerBase = getRegistry().getString( "appserver.base" );
groupsDirectory = new File( appServerBase + File.separatorChar + "groups" );
if ( !groupsDirectory.exists() )
{
groupsDirectory.mkdirs();
}
}
@Override
public File getMergedIndexDirectory( String repositoryGroupId )
{
return new File( groupsDirectory, repositoryGroupId );
}
public List<RepositoryGroup> getRepositoriesGroups()
throws RepositoryAdminException
{
@ -68,7 +89,8 @@ public class DefaultRepositoryGroupAdmin
{
repositoriesGroups.add( new RepositoryGroup( repositoryGroupConfiguration.getId(), new ArrayList<String>(
repositoryGroupConfiguration.getRepositories() ) ).mergedIndexPath(
repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl( repositoryGroupConfiguration.getMergedIndexTtl() ) );
repositoryGroupConfiguration.getMergedIndexPath() ).mergedIndexTtl(
repositoryGroupConfiguration.getMergedIndexTtl() ) );
}
return repositoriesGroups;
@ -285,7 +307,7 @@ public class DefaultRepositoryGroupAdmin
"Invalid character(s) found in identifier. Only the following characters are allowed: alphanumeric, '.', '-' and '_'" );
}
if ( repositoryGroup.getMergedIndexTtl() <= 0)
if ( repositoryGroup.getMergedIndexTtl() <= 0 )
{
throw new RepositoryAdminException( "Merged Index TTL must be greater than 0." );
}

73
archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java
View File

@ -19,6 +19,7 @@ package org.apache.archiva.webdav;
* under the License.
*/
import com.google.common.io.Files;
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.admin.model.beans.ManagedRepository;
import org.apache.archiva.admin.model.beans.RemoteRepository;
@ -37,6 +38,7 @@ import org.apache.archiva.configuration.RepositoryGroupConfiguration;
import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.merger.IndexMergerRequest;
import org.apache.archiva.indexer.merger.MergedRemoteIndexesTaskJob;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.apache.archiva.indexer.search.RepositorySearch;
import org.apache.archiva.maven2.metadata.MavenMetadataReader;
@ -115,7 +117,7 @@ import java.util.Set;
/**
*
*/
@Service( "davResourceFactory#archiva" )
@Service("davResourceFactory#archiva")
public class ArchivaDavResourceFactory
implements DavResourceFactory, Auditable
{
@ -134,7 +136,7 @@ public class ArchivaDavResourceFactory
private RepositoryRequest repositoryRequest;
@Inject
@Named( value = "repositoryProxyConnectors#default" )
@Named(value = "repositoryProxyConnectors#default")
private RepositoryProxyConnectors connectors;
@Inject
@ -149,7 +151,7 @@ public class ArchivaDavResourceFactory
private ServletAuthenticator servletAuth;
@Inject
@Named( value = "httpAuthenticator#basic" )
@Named(value = "httpAuthenticator#basic")
private HttpAuthenticator httpAuth;
@Inject
@ -176,11 +178,11 @@ public class ArchivaDavResourceFactory
private Digester digestMd5;
@Inject
@Named( value = "archivaTaskScheduler#repository" )
@Named(value = "archivaTaskScheduler#repository")
private RepositoryArchivaTaskScheduler scheduler;
@Inject
@Named(value= "fileLockManager#default")
@Named(value = "fileLockManager#default")
private FileLockManager fileLockManager;
private ApplicationContext applicationContext;
@ -345,12 +347,13 @@ public class ArchivaDavResourceFactory
if ( metadataChecksum.exists() )
{
LogicalResource logicalResource =
new LogicalResource (getLogicalResource( archivaLocator, null, false ) );
new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
resource =
new ArchivaDavResource( metadataChecksum.getAbsolutePath(), logicalResource.getPath(), null,
request.getRemoteAddr(), activePrincipal, request.getDavSession(),
archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager );
archivaLocator, this, mimeTypes, auditListeners, scheduler,
fileLockManager );
}
}
else
@ -383,7 +386,8 @@ public class ArchivaDavResourceFactory
{
File resourceFile = writeMergedMetadataToFile( mergedMetadata, filePath );
LogicalResource logicalResource = new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
LogicalResource logicalResource =
new LogicalResource( getLogicalResource( archivaLocator, null, false ) );
resource =
new ArchivaDavResource( resourceFile.getAbsolutePath(), logicalResource.getPath(), null,
@ -404,7 +408,8 @@ public class ArchivaDavResourceFactory
catch ( DigesterException de )
{
throw new DavException( HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
"Error occurred while generating checksum files." + de.getMessage() );
"Error occurred while generating checksum files."
+ de.getMessage() );
}
}
}
@ -445,7 +450,8 @@ public class ArchivaDavResourceFactory
File resourceFile = new File( temporaryIndexDirectory, requestedFileName );
resource = new ArchivaDavResource( resourceFile.getAbsolutePath(), requestedFileName, null,
request.getRemoteAddr(), activePrincipal, request.getDavSession(),
archivaLocator, this, mimeTypes, auditListeners, scheduler, fileLockManager );
archivaLocator, this, mimeTypes, auditListeners, scheduler,
fileLockManager );
}
else
@ -523,32 +529,39 @@ public class ArchivaDavResourceFactory
{
// FIXME remove this hack
// but currently managedRepository can be null in case of group
String layout = managedRepository == null ? new ManagedRepository( ).getLayout() : managedRepository.getLayout();
String layout = managedRepository == null ? new ManagedRepository().getLayout() : managedRepository.getLayout();
RepositoryStorage repositoryStorage =
this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class );
String path = repositoryStorage.getFilePath(
useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(), managedRepository );
useOrigResourcePath ? archivaLocator.getOrigResourcePath() : archivaLocator.getResourcePath(),
managedRepository );
log.debug( "found path {} for resourcePath: '{}' with managedRepo '{}' and layout '{}'", path,
archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(), layout );
archivaLocator.getResourcePath(), managedRepository == null ? "null" : managedRepository.getId(),
layout );
return path;
}
private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator, ManagedRepositoryContent managedRepositoryContent, String contextPath )
private String evaluatePathWithVersion( ArchivaDavResourceLocator archivaLocator,
ManagedRepositoryContent managedRepositoryContent, String contextPath )
throws DavException
{
String layout = managedRepositoryContent.getRepository() == null ? new ManagedRepository( ).getLayout() : managedRepositoryContent.getRepository().getLayout();
String layout = managedRepositoryContent.getRepository() == null
? new ManagedRepository().getLayout()
: managedRepositoryContent.getRepository().getLayout();
RepositoryStorage repositoryStorage =
this.applicationContext.getBean( "repositoryStorage#" + layout, RepositoryStorage.class );
try
{
return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(), managedRepositoryContent );
return repositoryStorage.getFilePathWithVersion( archivaLocator.getResourcePath(),
managedRepositoryContent );
}
catch ( RelocationException e )
{
String path = e.getPath();
log.debug( "Relocation to {}", path );
throw new BrowserRedirectException(contextPath + ( StringUtils.startsWith( path, "/" ) ? "": "/" ) + path, e.getRelocationType() );
throw new BrowserRedirectException( contextPath + ( StringUtils.startsWith( path, "/" ) ? "" : "/" ) + path,
e.getRelocationType() );
}
catch ( XMLException e )
{
@ -566,7 +579,7 @@ public class ArchivaDavResourceFactory
if ( isAuthorized( request, managedRepositoryContent.getId() ) )
{
// Maven Centric part ask evaluation if -SNAPSHOT
String path = evaluatePathWithVersion(archivaLocator, managedRepositoryContent, request.getContextPath());
String path = evaluatePathWithVersion( archivaLocator, managedRepositoryContent, request.getContextPath() );
if ( path.startsWith( "/" ) )
{
path = path.substring( 1 );
@ -1272,12 +1285,14 @@ public class ArchivaDavResourceFactory
}
Set<String> authzRepos = new HashSet<String>();
String permission = WebdavMethodUtil.getMethodPermission( request.getMethod() );
for ( String repository : repositories )
{
try
{
if ( servletAuth.isAuthorized( activePrincipal, repository,
WebdavMethodUtil.getMethodPermission( request.getMethod() ) ) )
if ( servletAuth.isAuthorized( activePrincipal, repository, permission ) )
{
authzRepos.add( repository );
authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) );
@ -1293,10 +1308,22 @@ public class ArchivaDavResourceFactory
}
log.info( "generate temporary merged index for repository group '{}' for repositories '{}'",
repositoryGroupConfiguration.getId(), authzRepos );
IndexingContext indexingContext = indexMerger.buildMergedIndex(
new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(),
File tempRepoFile = Files.createTempDir();
tempRepoFile.deleteOnExit();
IndexMergerRequest indexMergerRequest = new IndexMergerRequest( authzRepos, true, repositoryGroupConfiguration.getId(),
repositoryGroupConfiguration.getMergedIndexPath(),
repositoryGroupConfiguration.getMergedIndexTtl() ) );
repositoryGroupConfiguration.getMergedIndexTtl() ).mergedIndexDirectory(
tempRepoFile );
MergedRemoteIndexesTaskJob job = new MergedRemoteIndexesTaskJob();
MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest taskRequest =
new MergedRemoteIndexesTaskJob.MergedRemoteIndexesTaskRequest(indexMergerRequest, indexMerger);
IndexingContext indexingContext = job.execute( taskRequest ).getIndexingContext();
File mergedRepoDir = indexingContext.getIndexDirectoryFile();
TemporaryGroupIndex temporaryGroupIndex =
new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId(), repositoryGroupConfiguration.getId(),