Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[MRM-815] aggregate indices for repository groups. 

delete temporary on session end with a session listener
periodical taks to cleanup too old temp group index.

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1198011 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Olivier Lamy 2011-11-05 18:50:28 +00:00
parent e9d93aa21b
commit 7ad0e303a0
7 changed files with 325 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java
View File

@ -23,7 +23,6 @@ import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
import org.apache.commons.io.FileUtils;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
@ -31,14 +30,13 @@ import org.apache.maven.index.packer.IndexPacker;
import org.apache.maven.index.packer.IndexPackingRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
@ -62,7 +60,7 @@ public class DefaultIndexMerger
private IndexPacker indexPacker;
private List<TemporaryIndex> temporaryIndexes = new CopyOnWriteArrayList<TemporaryIndex>();
private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<TemporaryGroupIndex>();
@Inject
public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils )
@ -73,7 +71,7 @@ public class DefaultIndexMerger
indexPacker = plexusSisuBridge.lookup( IndexPacker.class, "default" );
}
public File buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
public IndexingContext buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
throws IndexMergerException
{
File tempRepoFile = Files.createTempDir();
@ -104,8 +102,8 @@ public class DefaultIndexMerger
IndexPackingRequest request = new IndexPackingRequest( indexingContext, indexLocation );
indexPacker.packIndex( request );
}
temporaryIndexes.add( new TemporaryIndex( tempRepoFile, tempRepoId ) );
return indexingContext.getIndexDirectoryFile();
temporaryGroupIndexes.add( new TemporaryGroupIndex( tempRepoFile, tempRepoId ) );
return indexingContext;
}
catch ( IOException e )
{
@ -117,70 +115,30 @@ public class DefaultIndexMerger
}
}
@Scheduled( fixedDelay = 900000 )
public void cleanTemporaryIndex()
@Async
public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex )
{
for ( TemporaryIndex temporaryIndex : temporaryIndexes )
if ( temporaryGroupIndex == null )
{
// cleanup files older than 30 minutes
if ( new Date().getTime() - temporaryIndex.creationTime > 1800000 )
return;
}
try
{
IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() );
if ( indexingContext != null )
{
try
{
IndexingContext context = indexer.getIndexingContexts().get( temporaryIndex.indexId );
if ( context != null )
{
indexer.removeIndexingContext( context, true );
}
else
{
FileUtils.deleteDirectory( temporaryIndex.directory );
}
temporaryIndexes.remove( temporaryIndex );
log.debug( "remove directory {}", temporaryIndex.directory );
}
catch ( IOException e )
{
log.warn( "failed to remove directory:" + temporaryIndex.directory, e );
}
indexer.removeIndexingContext( indexingContext, true );
}
temporaryIndexes.remove( temporaryIndex );
}
catch ( IOException e )
{
log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e );
}
}
private static class TemporaryIndex
public Collection<TemporaryGroupIndex> getTemporaryGroupIndexes()
{
private long creationTime = new Date().getTime();
private File directory;
private String indexId;
TemporaryIndex( File directory, String indexId )
{
this.directory = directory;
this.indexId = indexId;
}
@Override
public int hashCode()
{
return Long.toString( creationTime ).hashCode();
}
@Override
public boolean equals( Object o )
{
if ( this == o )
{
return true;
}
if ( !( o instanceof TemporaryIndex ) )
{
return false;
}
return this.creationTime == ( (TemporaryIndex) o ).creationTime;
}
return this.temporaryGroupIndexes;
}
}

15
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java
View File

@ -18,6 +18,8 @@ package org.apache.archiva.indexer.merger;
* under the License.
*/
import org.apache.maven.index.context.IndexingContext;
import java.io.File;
import java.util.Collection;
@ -27,12 +29,21 @@ import java.util.Collection;
*/
public interface IndexMerger
{
/**
* default tmp created group index ttl in minutes
*/
static final int DEFAULT_GROUP_INDEX_TTL = 1;
/**
* @param repositoriesIds repositories Ids to merge content
* @param packIndex will generate a downloadable index
* @param packIndex will generate a downloadable index
* @return a temporary directory with a merge index (directory marked deleteOnExit)
* @throws IndexMergerException
*/
File buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
IndexingContext buildMergedIndex( Collection<String> repositoriesIds, boolean packIndex )
throws IndexMergerException;
void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex );
Collection<TemporaryGroupIndex> getTemporaryGroupIndexes();
}

95
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java Normal file
View File

@ -0,0 +1,95 @@
package org.apache.archiva.indexer.merger;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.io.File;
import java.io.Serializable;
import java.util.Date;
/**
* @author Olivier Lamy
*/
public class TemporaryGroupIndex
implements Serializable
{
private long creationTime = new Date().getTime();
private File directory;
private String indexId;
public TemporaryGroupIndex( File directory, String indexId )
{
this.directory = directory;
this.indexId = indexId;
}
public long getCreationTime()
{
return creationTime;
}
public TemporaryGroupIndex setCreationTime( long creationTime )
{
this.creationTime = creationTime;
return this;
}
public File getDirectory()
{
return directory;
}
public TemporaryGroupIndex setDirectory( File directory )
{
this.directory = directory;
return this;
}
public String getIndexId()
{
return indexId;
}
public TemporaryGroupIndex setIndexId( String indexId )
{
this.indexId = indexId;
return this;
}
@Override
public int hashCode()
{
return Long.toString( creationTime ).hashCode();
}
@Override
public boolean equals( Object o )
{
if ( this == o )
{
return true;
}
if ( !( o instanceof TemporaryGroupIndex ) )
{
return false;
}
return this.creationTime == ( (TemporaryGroupIndex) o ).creationTime;
}
}

85
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java Normal file
View File

@ -0,0 +1,85 @@
package org.apache.archiva.indexer.merger;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
import java.io.IOException;
import java.util.Date;
/**
* @author Olivier Lamy
* @since 1.4-M2
*/
@Service
public class TemporaryGroupIndexCleaner
{
private Logger log = LoggerFactory.getLogger( getClass() );
@Inject
private IndexMerger indexMerger;
private NexusIndexer indexer;
@Inject
public TemporaryGroupIndexCleaner( PlexusSisuBridge plexusSisuBridge )
throws PlexusSisuBridgeException
{
indexer = plexusSisuBridge.lookup( NexusIndexer.class );
}
// 900000
@Scheduled( fixedDelay = 900000 )
public void cleanTemporaryIndex()
{
for ( TemporaryGroupIndex temporaryGroupIndex : indexMerger.getTemporaryGroupIndexes() )
{
// cleanup files older than 60 minutes 3600000
if ( new Date().getTime() - temporaryGroupIndex.getCreationTime() > 3600000 )
{
try
{
IndexingContext context = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() );
if ( context != null )
{
indexer.removeIndexingContext( context, true );
}
else
{
indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex );
}
indexMerger.getTemporaryGroupIndexes().remove( temporaryGroupIndex );
log.debug( "remove directory {}", temporaryGroupIndex.getDirectory() );
}
catch ( IOException e )
{
log.warn( "failed to remove directory:" + temporaryGroupIndex.getDirectory(), e );
}
}
}
}
}

5
archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/web.xml
View File

@ -89,6 +89,11 @@
<listener-class>net.sf.ehcache.constructs.web.ShutdownListener</listener-class>
</listener>
<!-- to cleanup temporary group index created during a session -->
<listener>
<listener-class>org.apache.archiva.webdav.util.TemporaryGroupIndexSessionCleaner</listener-class>
</listener>
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>

41
archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java
View File

@ -31,6 +31,7 @@ import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.configuration.RepositoryGroupConfiguration;
import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.apache.archiva.indexer.search.RepositorySearch;
import org.apache.archiva.model.ArchivaRepositoryMetadata;
import org.apache.archiva.model.ArtifactReference;
@ -52,6 +53,7 @@ import org.apache.archiva.scheduler.repository.RepositoryArchivaTaskScheduler;
import org.apache.archiva.security.ServletAuthenticator;
import org.apache.archiva.webdav.util.MimeTypes;
import org.apache.archiva.webdav.util.RepositoryPathUtil;
import org.apache.archiva.webdav.util.TemporaryGroupIndexSessionCleaner;
import org.apache.archiva.webdav.util.WebdavMethodUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
@ -65,6 +67,7 @@ import org.apache.jackrabbit.webdav.DavServletResponse;
import org.apache.jackrabbit.webdav.DavSession;
import org.apache.jackrabbit.webdav.lock.LockManager;
import org.apache.jackrabbit.webdav.lock.SimpleLockManager;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.model.DistributionManagement;
import org.apache.maven.model.Model;
import org.apache.maven.model.Relocation;
@ -98,6 +101,7 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -1198,16 +1202,29 @@ public class ArchivaDavResourceFactory
try
{
HttpSession session = request.getSession();
Map<String, File> testValue = (Map<String, File>) session.getAttribute( "TMP_GROUP_INDEXES" );
if ( testValue == null )
Map<String, TemporaryGroupIndex> temporaryGroupIndexMap =
(Map<String, TemporaryGroupIndex>) session.getAttribute(
TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY );
if ( temporaryGroupIndexMap == null )
{
testValue = new HashMap<String, File>();
temporaryGroupIndexMap = new HashMap<String, TemporaryGroupIndex>();
}
File tmp = testValue.get( groupId );
if ( tmp != null )
TemporaryGroupIndex tmp = temporaryGroupIndexMap.get( groupId );
if ( tmp != null && tmp.getDirectory() != null && tmp.getDirectory().exists() )
{
return tmp;
if ( System.currentTimeMillis() - tmp.getCreationTime() > ( IndexMerger.DEFAULT_GROUP_INDEX_TTL * 60
* 1000 ) )
{
log.debug( "tmp group index is too old so delete it" );
indexMerger.cleanTemporaryGroupIndex( tmp );
}
else
{
return tmp.getDirectory();
}
}
Set<String> authzRepos = new HashSet<String>();
@ -1232,10 +1249,14 @@ public class ArchivaDavResourceFactory
}
}
}
File mergedRepoDir = indexMerger.buildMergedIndex( authzRepos, true );
testValue.put( groupId, mergedRepoDir );
session.setAttribute( "TMP_GROUP_INDEXES", testValue );
IndexingContext indexingContext = indexMerger.buildMergedIndex( authzRepos, true );
File mergedRepoDir = indexingContext.getIndexDirectoryFile();
TemporaryGroupIndex temporaryGroupIndex =
new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId() ).setCreationTime(
new Date().getTime() );
temporaryGroupIndexMap.put( groupId, temporaryGroupIndex );
session.setAttribute( TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY,
temporaryGroupIndexMap );
return mergedRepoDir;
}
catch ( RepositoryAdminException e )

74
archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/util/TemporaryGroupIndexSessionCleaner.java Normal file
View File

@ -0,0 +1,74 @@
package org.apache.archiva.webdav.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.context.WebApplicationContext;
import org.springframework.web.context.support.WebApplicationContextUtils;
import javax.servlet.http.HttpSessionEvent;
import javax.servlet.http.HttpSessionListener;
import java.util.HashMap;
import java.util.Map;
/**
* this http session listener will delete repository group index requested by a user
* at this end of the http session
*
* @author Olivier Lamy
* @since 1.4-M2
*/
public class TemporaryGroupIndexSessionCleaner
implements HttpSessionListener
{
private Logger log = LoggerFactory.getLogger( getClass() );
private IndexMerger indexMerger;
public static final String TEMPORARY_INDEX_SESSION_KEY = TemporaryGroupIndexSessionCleaner.class.getName();
public void sessionCreated( HttpSessionEvent httpSessionEvent )
{
// ensure the map is here to avoid NPE
httpSessionEvent.getSession().setAttribute( TEMPORARY_INDEX_SESSION_KEY,
new HashMap<String, TemporaryGroupIndex>() );
WebApplicationContext webApplicationContext = WebApplicationContextUtils.getRequiredWebApplicationContext(
httpSessionEvent.getSession().getServletContext() );
indexMerger = webApplicationContext.getBean( IndexMerger.class );
}
public void sessionDestroyed( HttpSessionEvent httpSessionEvent )
{
Map<String, TemporaryGroupIndex> tempFilesPerKey =
(Map<String, TemporaryGroupIndex>) httpSessionEvent.getSession().getAttribute(
TEMPORARY_INDEX_SESSION_KEY );
for ( TemporaryGroupIndex temporaryGroupIndex : tempFilesPerKey.values() )
{
log.info( "cleanup temporaryGroupIndex {} directory {}", temporaryGroupIndex.getIndexId(),
temporaryGroupIndex.getDirectory().getAbsolutePath() );
indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex );
}
}
}
}