From 7ad0e303a0b77c9abb8197b4dd528c6f90bd74b3 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Sat, 5 Nov 2011 18:50:28 +0000 Subject: [PATCH] [MRM-815] aggregate indices for repository groups. delete temporary on session end with a session listener periodical taks to cleanup too old temp group index. git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1198011 13f79535-47bb-0310-9956-ffa450edef68 --- .../indexer/merger/DefaultIndexMerger.java | 86 +++++------------ .../archiva/indexer/merger/IndexMerger.java | 15 ++- .../indexer/merger/TemporaryGroupIndex.java | 95 +++++++++++++++++++ .../merger/TemporaryGroupIndexCleaner.java | 85 +++++++++++++++++ .../src/main/webapp/WEB-INF/web.xml | 5 + .../webdav/ArchivaDavResourceFactory.java | 41 ++++++-- .../TemporaryGroupIndexSessionCleaner.java | 74 +++++++++++++++ 7 files changed, 325 insertions(+), 76 deletions(-) create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java create mode 100644 archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/util/TemporaryGroupIndexSessionCleaner.java diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java index 9c0613490..674f261fb 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java @@ -23,7 +23,6 @@ import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; import org.apache.archiva.common.plexusbridge.MavenIndexerUtils; import org.apache.archiva.common.plexusbridge.PlexusSisuBridge; import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; -import org.apache.commons.io.FileUtils; import org.apache.maven.index.NexusIndexer; import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; @@ -31,14 +30,13 @@ import org.apache.maven.index.packer.IndexPacker; import org.apache.maven.index.packer.IndexPackingRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.util.Collection; -import java.util.Date; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -62,7 +60,7 @@ public class DefaultIndexMerger private IndexPacker indexPacker; - private List temporaryIndexes = new CopyOnWriteArrayList(); + private List temporaryGroupIndexes = new CopyOnWriteArrayList(); @Inject public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils ) @@ -73,7 +71,7 @@ public class DefaultIndexMerger indexPacker = plexusSisuBridge.lookup( IndexPacker.class, "default" ); } - public File buildMergedIndex( Collection repositoriesIds, boolean packIndex ) + public IndexingContext buildMergedIndex( Collection repositoriesIds, boolean packIndex ) throws IndexMergerException { File tempRepoFile = Files.createTempDir(); @@ -104,8 +102,8 @@ public class DefaultIndexMerger IndexPackingRequest request = new IndexPackingRequest( indexingContext, indexLocation ); indexPacker.packIndex( request ); } - temporaryIndexes.add( new TemporaryIndex( tempRepoFile, tempRepoId ) ); - return indexingContext.getIndexDirectoryFile(); + temporaryGroupIndexes.add( new TemporaryGroupIndex( tempRepoFile, tempRepoId ) ); + return indexingContext; } catch ( IOException e ) { @@ -117,70 +115,30 @@ public class DefaultIndexMerger } } - - @Scheduled( fixedDelay = 900000 ) - public void cleanTemporaryIndex() + @Async + public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex ) { - for ( TemporaryIndex temporaryIndex : temporaryIndexes ) + if ( temporaryGroupIndex == null ) { - // cleanup files older than 30 minutes - if ( new Date().getTime() - temporaryIndex.creationTime > 1800000 ) + return; + } + + try + { + IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() ); + if ( indexingContext != null ) { - try - { - IndexingContext context = indexer.getIndexingContexts().get( temporaryIndex.indexId ); - if ( context != null ) - { - indexer.removeIndexingContext( context, true ); - } - else - { - FileUtils.deleteDirectory( temporaryIndex.directory ); - } - temporaryIndexes.remove( temporaryIndex ); - log.debug( "remove directory {}", temporaryIndex.directory ); - } - catch ( IOException e ) - { - log.warn( "failed to remove directory:" + temporaryIndex.directory, e ); - } + indexer.removeIndexingContext( indexingContext, true ); } - temporaryIndexes.remove( temporaryIndex ); + } + catch ( IOException e ) + { + log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e ); } } - private static class TemporaryIndex + public Collection getTemporaryGroupIndexes() { - private long creationTime = new Date().getTime(); - - private File directory; - - private String indexId; - - TemporaryIndex( File directory, String indexId ) - { - this.directory = directory; - this.indexId = indexId; - } - - @Override - public int hashCode() - { - return Long.toString( creationTime ).hashCode(); - } - - @Override - public boolean equals( Object o ) - { - if ( this == o ) - { - return true; - } - if ( !( o instanceof TemporaryIndex ) ) - { - return false; - } - return this.creationTime == ( (TemporaryIndex) o ).creationTime; - } + return this.temporaryGroupIndexes; } } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java index 89e2c7d6f..17a1f21f6 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java @@ -18,6 +18,8 @@ package org.apache.archiva.indexer.merger; * under the License. */ +import org.apache.maven.index.context.IndexingContext; + import java.io.File; import java.util.Collection; @@ -27,12 +29,21 @@ import java.util.Collection; */ public interface IndexMerger { + /** + * default tmp created group index ttl in minutes + */ + static final int DEFAULT_GROUP_INDEX_TTL = 1; + /** * @param repositoriesIds repositories Ids to merge content - * @param packIndex will generate a downloadable index + * @param packIndex will generate a downloadable index * @return a temporary directory with a merge index (directory marked deleteOnExit) * @throws IndexMergerException */ - File buildMergedIndex( Collection repositoriesIds, boolean packIndex ) + IndexingContext buildMergedIndex( Collection repositoriesIds, boolean packIndex ) throws IndexMergerException; + + void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex ); + + Collection getTemporaryGroupIndexes(); } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java new file mode 100644 index 000000000..489ac1594 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndex.java @@ -0,0 +1,95 @@ +package org.apache.archiva.indexer.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.io.Serializable; +import java.util.Date; + +/** + * @author Olivier Lamy + */ +public class TemporaryGroupIndex + implements Serializable +{ + private long creationTime = new Date().getTime(); + + private File directory; + + private String indexId; + + public TemporaryGroupIndex( File directory, String indexId ) + { + this.directory = directory; + this.indexId = indexId; + } + + public long getCreationTime() + { + return creationTime; + } + + public TemporaryGroupIndex setCreationTime( long creationTime ) + { + this.creationTime = creationTime; + return this; + } + + public File getDirectory() + { + return directory; + } + + public TemporaryGroupIndex setDirectory( File directory ) + { + this.directory = directory; + return this; + } + + public String getIndexId() + { + return indexId; + } + + public TemporaryGroupIndex setIndexId( String indexId ) + { + this.indexId = indexId; + return this; + } + + @Override + public int hashCode() + { + return Long.toString( creationTime ).hashCode(); + } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( !( o instanceof TemporaryGroupIndex ) ) + { + return false; + } + return this.creationTime == ( (TemporaryGroupIndex) o ).creationTime; + } +} diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java new file mode 100644 index 000000000..0845db6b1 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/TemporaryGroupIndexCleaner.java @@ -0,0 +1,85 @@ +package org.apache.archiva.indexer.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.common.plexusbridge.PlexusSisuBridge; +import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; +import org.apache.maven.index.NexusIndexer; +import org.apache.maven.index.context.IndexingContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.Date; + +/** + * @author Olivier Lamy + * @since 1.4-M2 + */ +@Service +public class TemporaryGroupIndexCleaner +{ + private Logger log = LoggerFactory.getLogger( getClass() ); + + @Inject + private IndexMerger indexMerger; + + private NexusIndexer indexer; + + @Inject + public TemporaryGroupIndexCleaner( PlexusSisuBridge plexusSisuBridge ) + throws PlexusSisuBridgeException + { + indexer = plexusSisuBridge.lookup( NexusIndexer.class ); + } + + // 900000 + @Scheduled( fixedDelay = 900000 ) + public void cleanTemporaryIndex() + { + for ( TemporaryGroupIndex temporaryGroupIndex : indexMerger.getTemporaryGroupIndexes() ) + { + // cleanup files older than 60 minutes 3600000 + if ( new Date().getTime() - temporaryGroupIndex.getCreationTime() > 3600000 ) + { + try + { + IndexingContext context = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() ); + if ( context != null ) + { + indexer.removeIndexingContext( context, true ); + } + else + { + indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex ); + } + indexMerger.getTemporaryGroupIndexes().remove( temporaryGroupIndex ); + log.debug( "remove directory {}", temporaryGroupIndex.getDirectory() ); + } + catch ( IOException e ) + { + log.warn( "failed to remove directory:" + temporaryGroupIndex.getDirectory(), e ); + } + } + } + } +} diff --git a/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/web.xml b/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/web.xml index 2cb7f1d9d..084c7ff39 100644 --- a/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/web.xml +++ b/archiva-modules/archiva-web/archiva-webapp/src/main/webapp/WEB-INF/web.xml @@ -89,6 +89,11 @@ net.sf.ehcache.constructs.web.ShutdownListener + + + org.apache.archiva.webdav.util.TemporaryGroupIndexSessionCleaner + + contextConfigLocation diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java index f57f64b80..a4341ced0 100644 --- a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java @@ -31,6 +31,7 @@ import org.apache.archiva.configuration.ArchivaConfiguration; import org.apache.archiva.configuration.RepositoryGroupConfiguration; import org.apache.archiva.indexer.merger.IndexMerger; import org.apache.archiva.indexer.merger.IndexMergerException; +import org.apache.archiva.indexer.merger.TemporaryGroupIndex; import org.apache.archiva.indexer.search.RepositorySearch; import org.apache.archiva.model.ArchivaRepositoryMetadata; import org.apache.archiva.model.ArtifactReference; @@ -52,6 +53,7 @@ import org.apache.archiva.scheduler.repository.RepositoryArchivaTaskScheduler; import org.apache.archiva.security.ServletAuthenticator; import org.apache.archiva.webdav.util.MimeTypes; import org.apache.archiva.webdav.util.RepositoryPathUtil; +import org.apache.archiva.webdav.util.TemporaryGroupIndexSessionCleaner; import org.apache.archiva.webdav.util.WebdavMethodUtil; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; @@ -65,6 +67,7 @@ import org.apache.jackrabbit.webdav.DavServletResponse; import org.apache.jackrabbit.webdav.DavSession; import org.apache.jackrabbit.webdav.lock.LockManager; import org.apache.jackrabbit.webdav.lock.SimpleLockManager; +import org.apache.maven.index.context.IndexingContext; import org.apache.maven.model.DistributionManagement; import org.apache.maven.model.Model; import org.apache.maven.model.Relocation; @@ -98,6 +101,7 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -1198,16 +1202,29 @@ public class ArchivaDavResourceFactory try { HttpSession session = request.getSession(); - Map testValue = (Map) session.getAttribute( "TMP_GROUP_INDEXES" ); - if ( testValue == null ) + + Map temporaryGroupIndexMap = + (Map) session.getAttribute( + TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY ); + if ( temporaryGroupIndexMap == null ) { - testValue = new HashMap(); + temporaryGroupIndexMap = new HashMap(); } - File tmp = testValue.get( groupId ); - if ( tmp != null ) + TemporaryGroupIndex tmp = temporaryGroupIndexMap.get( groupId ); + + if ( tmp != null && tmp.getDirectory() != null && tmp.getDirectory().exists() ) { - return tmp; + if ( System.currentTimeMillis() - tmp.getCreationTime() > ( IndexMerger.DEFAULT_GROUP_INDEX_TTL * 60 + * 1000 ) ) + { + log.debug( "tmp group index is too old so delete it" ); + indexMerger.cleanTemporaryGroupIndex( tmp ); + } + else + { + return tmp.getDirectory(); + } } Set authzRepos = new HashSet(); @@ -1232,10 +1249,14 @@ public class ArchivaDavResourceFactory } } } - - File mergedRepoDir = indexMerger.buildMergedIndex( authzRepos, true ); - testValue.put( groupId, mergedRepoDir ); - session.setAttribute( "TMP_GROUP_INDEXES", testValue ); + IndexingContext indexingContext = indexMerger.buildMergedIndex( authzRepos, true ); + File mergedRepoDir = indexingContext.getIndexDirectoryFile(); + TemporaryGroupIndex temporaryGroupIndex = + new TemporaryGroupIndex( mergedRepoDir, indexingContext.getId() ).setCreationTime( + new Date().getTime() ); + temporaryGroupIndexMap.put( groupId, temporaryGroupIndex ); + session.setAttribute( TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY, + temporaryGroupIndexMap ); return mergedRepoDir; } catch ( RepositoryAdminException e ) diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/util/TemporaryGroupIndexSessionCleaner.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/util/TemporaryGroupIndexSessionCleaner.java new file mode 100644 index 000000000..20ad746bd --- /dev/null +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/util/TemporaryGroupIndexSessionCleaner.java @@ -0,0 +1,74 @@ +package org.apache.archiva.webdav.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.indexer.merger.IndexMerger; +import org.apache.archiva.indexer.merger.TemporaryGroupIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.context.WebApplicationContext; +import org.springframework.web.context.support.WebApplicationContextUtils; + +import javax.servlet.http.HttpSessionEvent; +import javax.servlet.http.HttpSessionListener; +import java.util.HashMap; +import java.util.Map; + +/** + * this http session listener will delete repository group index requested by a user + * at this end of the http session + * + * @author Olivier Lamy + * @since 1.4-M2 + */ +public class TemporaryGroupIndexSessionCleaner + implements HttpSessionListener +{ + + private Logger log = LoggerFactory.getLogger( getClass() ); + + private IndexMerger indexMerger; + + public static final String TEMPORARY_INDEX_SESSION_KEY = TemporaryGroupIndexSessionCleaner.class.getName(); + + public void sessionCreated( HttpSessionEvent httpSessionEvent ) + { + // ensure the map is here to avoid NPE + httpSessionEvent.getSession().setAttribute( TEMPORARY_INDEX_SESSION_KEY, + new HashMap() ); + WebApplicationContext webApplicationContext = WebApplicationContextUtils.getRequiredWebApplicationContext( + httpSessionEvent.getSession().getServletContext() ); + indexMerger = webApplicationContext.getBean( IndexMerger.class ); + } + + public void sessionDestroyed( HttpSessionEvent httpSessionEvent ) + { + Map tempFilesPerKey = + (Map) httpSessionEvent.getSession().getAttribute( + TEMPORARY_INDEX_SESSION_KEY ); + + for ( TemporaryGroupIndex temporaryGroupIndex : tempFilesPerKey.values() ) + { + log.info( "cleanup temporaryGroupIndex {} directory {}", temporaryGroupIndex.getIndexId(), + temporaryGroupIndex.getDirectory().getAbsolutePath() ); + indexMerger.cleanTemporaryGroupIndex( temporaryGroupIndex ); + } + } +} +}