From 0262091faaaed893c6c43d2a7090d5d92cbba4aa Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Wed, 2 Nov 2011 22:31:42 +0000 Subject: [PATCH] [MRM-815] aggregate indices for repository groups. git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1196840 13f79535-47bb-0310-9956-ffa450edef68 --- .../archiva-base/archiva-indexer/pom.xml | 4 + .../indexer/merger/DefaultIndexMerger.java | 94 +++++++++++++++++++ .../archiva/indexer/merger/IndexMerger.java | 37 ++++++++ .../indexer/merger/IndexMergerException.java | 32 +++++++ .../indexer/search/NexusRepositorySearch.java | 8 +- .../indexer/search/RepositorySearch.java | 16 +++- .../resources/META-INF/spring-context.xml | 2 +- .../DefaultDownloadRemoteIndexScheduler.java | 87 ++++++----------- .../archiva-web/archiva-webdav/pom.xml | 4 + .../webdav/ArchivaDavResourceFactory.java | 57 +++++++++++ pom.xml | 1 - 11 files changed, 274 insertions(+), 68 deletions(-) create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java create mode 100644 archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerException.java diff --git a/archiva-modules/archiva-base/archiva-indexer/pom.xml b/archiva-modules/archiva-base/archiva-indexer/pom.xml index 5462d490f..0d5b87a4f 100644 --- a/archiva-modules/archiva-base/archiva-indexer/pom.xml +++ b/archiva-modules/archiva-base/archiva-indexer/pom.xml @@ -45,6 +45,10 @@ org.codehaus.plexus plexus-utils + + com.google.guava + guava + org.codehaus.plexus plexus-digest diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java new file mode 100644 index 000000000..2a3411a16 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java @@ -0,0 +1,94 @@ +package org.apache.archiva.indexer.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.io.Files; +import com.google.inject.Inject; +import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; +import org.apache.archiva.common.plexusbridge.MavenIndexerUtils; +import org.apache.archiva.common.plexusbridge.PlexusSisuBridge; +import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; +import org.apache.maven.index.NexusIndexer; +import org.apache.maven.index.context.IndexingContext; +import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +/** + * @author Olivier Lamy + * @since 1.4-M2 + */ +@Service( "indexMerger#default" ) +public class DefaultIndexMerger + implements IndexMerger +{ + + @Inject + private ManagedRepositoryAdmin managedRepositoryAdmin; + + private MavenIndexerUtils mavenIndexerUtils; + + private NexusIndexer indexer; + + @javax.inject.Inject + public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils ) + throws PlexusSisuBridgeException + { + this.indexer = plexusSisuBridge.lookup( NexusIndexer.class ); + this.mavenIndexerUtils = mavenIndexerUtils; + } + + public File buildMergedIndex( Collection repositoriesIds ) + throws IndexMergerException + { + File tempRepoFile = Files.createTempDir(); + tempRepoFile.deleteOnExit(); + + String tempRepoId = tempRepoFile.getName(); + + try + { + IndexingContext indexingContext = + indexer.addIndexingContext( tempRepoId, tempRepoId, tempRepoFile, new File( tempRepoFile, ".indexer" ), + null, null, mavenIndexerUtils.getAllIndexCreators() ); + + for ( String repoId : repositoriesIds ) + { + IndexingContext idxToMerge = indexer.getIndexingContexts().get( repoId ); + if ( idxToMerge != null ) + { + indexingContext.merge( idxToMerge.getIndexDirectory() ); + } + } + + return indexingContext.getIndexDirectoryFile(); + } + catch ( IOException e ) + { + throw new IndexMergerException( e.getMessage(), e ); + } + catch ( UnsupportedExistingLuceneIndexException e ) + { + throw new IndexMergerException( e.getMessage(), e ); + } + } +} diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java new file mode 100644 index 000000000..7f6333f37 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java @@ -0,0 +1,37 @@ +package org.apache.archiva.indexer.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.util.Collection; + +/** + * @author Olivier Lamy + * @since 1.4-M2 + */ +public interface IndexMerger +{ + /** + * @param repositoriesIds repositories Ids to merge content + * @return a temporary directory with a merge index (directory marked deleteOnExit) + * @throws IndexMergerException + */ + File buildMergedIndex( Collection repositoriesIds ) + throws IndexMergerException; +} diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerException.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerException.java new file mode 100644 index 000000000..3930831d6 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerException.java @@ -0,0 +1,32 @@ +package org.apache.archiva.indexer.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * @author Olivier Lamy + * @since 1.4-M2 + */ +public class IndexMergerException + extends Exception +{ + public IndexMergerException( String message, Throwable t ) + { + super( message, t ); + } +} diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java index 3e284e438..0ba4559bd 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java @@ -328,6 +328,10 @@ public class NexusRepositorySearch if ( indexDir != null && !"".equals( indexDir ) ) { indexDirectory = new File( repoConfig.getIndexDirectory() ); + if ( !indexDirectory.isAbsolute() ) + { + indexDirectory = new File( repoConfig.getLocation(), repoConfig.getIndexDirectory() ); + } } else { @@ -388,7 +392,7 @@ public class NexusRepositorySearch } - private Set getRemoteIndexingContextIds( String managedRepoId ) + public Set getRemoteIndexingContextIds( String managedRepoId ) throws RepositoryAdminException { Set ids = new HashSet(); @@ -674,4 +678,6 @@ public class NexusRepositorySearch return paginated; } + + } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/RepositorySearch.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/RepositorySearch.java index ccfc133de..e2a541e00 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/RepositorySearch.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/RepositorySearch.java @@ -19,29 +19,32 @@ package org.apache.archiva.indexer.search; * under the License. */ +import org.apache.archiva.admin.model.RepositoryAdminException; + import java.util.Collection; import java.util.List; +import java.util.Set; public interface RepositorySearch { /** * Quick search. - * + * * @param principal * @param selectedRepos * @param term * @param limits - * @param previousSearchTerms + * @param previousSearchTerms * @return */ SearchResults search( String principal, List selectedRepos, String term, SearchResultLimits limits, List previousSearchTerms ) throws RepositorySearchException; - + /** * Advanced search. - * + * * @param principal * @param searchFields * @param limits @@ -49,7 +52,10 @@ public interface RepositorySearch */ SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits ) throws RepositorySearchException; - + Collection getAllGroupIds( String principal, List selectedRepos ) throws RepositorySearchException; + + Set getRemoteIndexingContextIds( String managedRepoId ) + throws RepositoryAdminException; } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml b/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml index 39041ba0a..e23b9807e 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml @@ -29,7 +29,7 @@ default-lazy-init="true"> - + org.sonatype.nexus.index.DefaultNexusIndexer diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java index 18a5a98e3..83439050b 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java @@ -47,7 +47,6 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Named; -import java.io.File; import java.io.IOException; import java.util.Date; import java.util.List; @@ -62,7 +61,7 @@ public class DefaultDownloadRemoteIndexScheduler implements ConfigurationListener, DownloadRemoteIndexScheduler { - private Logger log = LoggerFactory.getLogger( getClass( ) ); + private Logger log = LoggerFactory.getLogger( getClass() ); @Inject @Named( value = "taskScheduler#indexDownloadRemote" ) @@ -94,50 +93,35 @@ public class DefaultDownloadRemoteIndexScheduler private IndexUpdater indexUpdater; // store ids about currently running remote download : updated in DownloadRemoteIndexTask - private List runningRemoteDownloadIds = new CopyOnWriteArrayList( ); + private List runningRemoteDownloadIds = new CopyOnWriteArrayList(); @PostConstruct - public void startup( ) + public void startup() throws ArchivaException, RepositoryAdminException, PlexusSisuBridgeException, IOException, UnsupportedExistingLuceneIndexException, DownloadRemoteIndexException { archivaConfiguration.addListener( this ); // TODO add indexContexts even if null - // FIXME get this from ArchivaAdministration - String appServerBase = System.getProperty( "appserver.base" ); - nexusIndexer = plexusSisuBridge.lookup( NexusIndexer.class ); indexUpdater = plexusSisuBridge.lookup( IndexUpdater.class ); - for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories( ) ) + for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() ) { - String contextKey = "remote-" + remoteRepository.getId( ); - if ( nexusIndexer.getIndexingContexts( ).get( contextKey ) != null ) + String contextKey = "remote-" + remoteRepository.getId(); + IndexingContext context = nexusIndexer.getIndexingContexts().get( contextKey ); + if ( context == null ) { continue; } - // create path - File repoDir = new File( appServerBase, "data/remotes/" + remoteRepository.getId( ) ); - if ( !repoDir.exists( ) ) - { - repoDir.mkdirs( ); - } - File indexDirectory = new File( repoDir, ".index" ); - if ( !indexDirectory.exists( ) ) - { - indexDirectory.mkdirs( ); - } - nexusIndexer.addIndexingContext( contextKey, remoteRepository.getId( ), repoDir, indexDirectory, - remoteRepository.getUrl( ), calculateIndexRemoteUrl( remoteRepository ), - mavenIndexerUtils.getAllIndexCreators( ) ); + // TODO record jobs from configuration - if ( remoteRepository.isDownloadRemoteIndex( ) && StringUtils.isNotEmpty( - remoteRepository.getCronExpression( ) ) ) + if ( remoteRepository.isDownloadRemoteIndex() && StringUtils.isNotEmpty( + remoteRepository.getCronExpression() ) ) { - boolean fullDownload = indexDirectory.list( ).length == 0; - scheduleDownloadRemote( remoteRepository.getId( ), false, fullDownload ); + boolean fullDownload = context.getIndexDirectoryFile().list().length == 0; + scheduleDownloadRemote( remoteRepository.getId(), false, fullDownload ); } } @@ -145,13 +129,13 @@ public class DefaultDownloadRemoteIndexScheduler } @PreDestroy - public void shutdown( ) + public void shutdown() throws RepositoryAdminException, IOException { - for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories( ) ) + for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() ) { - String contextKey = "remote-" + remoteRepository.getId( ); - IndexingContext context = nexusIndexer.getIndexingContexts( ).get( contextKey ); + String contextKey = "remote-" + remoteRepository.getId(); + IndexingContext context = nexusIndexer.getIndexingContexts().get( contextKey ); if ( context == null ) { continue; @@ -178,65 +162,48 @@ public class DefaultDownloadRemoteIndexScheduler return; } NetworkProxy networkProxy = null; - if ( StringUtils.isNotBlank( remoteRepository.getRemoteDownloadNetworkProxyId( ) ) ) + if ( StringUtils.isNotBlank( remoteRepository.getRemoteDownloadNetworkProxyId() ) ) { - networkProxy = networkProxyAdmin.getNetworkProxy( remoteRepository.getRemoteDownloadNetworkProxyId( ) ); + networkProxy = networkProxyAdmin.getNetworkProxy( remoteRepository.getRemoteDownloadNetworkProxyId() ); if ( networkProxy == null ) { log.warn( "your remote repository is configured to download remote index trought a proxy we cannot find id:{}", - remoteRepository.getRemoteDownloadNetworkProxyId( ) ); + remoteRepository.getRemoteDownloadNetworkProxyId() ); } } DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest = - new DownloadRemoteIndexTaskRequest( ).setRemoteRepository( remoteRepository ).setNetworkProxy( + new DownloadRemoteIndexTaskRequest().setRemoteRepository( remoteRepository ).setNetworkProxy( networkProxy ).setFullDownload( fullDownload ).setWagonFactory( wagonFactory ).setNexusIndexer( nexusIndexer ).setIndexUpdater( indexUpdater ); if ( now ) { - log.info( "schedule download remote index for repository {}", remoteRepository.getId( ) ); + log.info( "schedule download remote index for repository {}", remoteRepository.getId() ); // do it in async taskScheduler.schedule( new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ), - new Date( ) ); + new Date() ); } else { log.info( "schedule download remote index for repository {} with cron expression {}", - remoteRepository.getId( ), remoteRepository.getCronExpression( ) ); + remoteRepository.getId(), remoteRepository.getCronExpression() ); taskScheduler.schedule( new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ), - new CronTrigger( remoteRepository.getCronExpression( ) ) ); + new CronTrigger( remoteRepository.getCronExpression() ) ); } } catch ( RepositoryAdminException e ) { - log.error( e.getMessage( ), e ); - throw new DownloadRemoteIndexException( e.getMessage( ), e ); + log.error( e.getMessage(), e ); + throw new DownloadRemoteIndexException( e.getMessage(), e ); } } - protected String calculateIndexRemoteUrl( RemoteRepository remoteRepository ) - { - if ( StringUtils.startsWith( remoteRepository.getRemoteIndexUrl( ), "http" ) ) - { - String baseUrl = remoteRepository.getRemoteIndexUrl( ); - return baseUrl.endsWith( "/" ) ? StringUtils.substringBeforeLast( baseUrl, "/" ) : baseUrl; - } - String baseUrl = StringUtils.endsWith( remoteRepository.getUrl( ), "/" ) ? StringUtils.substringBeforeLast( - remoteRepository.getUrl( ), "/" ) : remoteRepository.getUrl( ); - - baseUrl = StringUtils.isEmpty( remoteRepository.getRemoteIndexUrl( ) ) - ? baseUrl + "/.index" - : baseUrl + "/" + remoteRepository.getRemoteIndexUrl( ); - return baseUrl; - - } - - public TaskScheduler getTaskScheduler( ) + public TaskScheduler getTaskScheduler() { return taskScheduler; } diff --git a/archiva-modules/archiva-web/archiva-webdav/pom.xml b/archiva-modules/archiva-web/archiva-webdav/pom.xml index e37ad2895..5ca1c57e2 100644 --- a/archiva-modules/archiva-web/archiva-webdav/pom.xml +++ b/archiva-modules/archiva-web/archiva-webdav/pom.xml @@ -66,6 +66,10 @@ org.apache.archiva archiva-security + + org.apache.archiva + archiva-indexer + org.apache.jackrabbit jackrabbit-webdav diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java index b99b2cf4d..03cf17dfc 100644 --- a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java @@ -19,6 +19,7 @@ package org.apache.archiva.webdav; * under the License. */ +import org.apache.archiva.admin.model.RepositoryAdminException; import org.apache.archiva.audit.AuditEvent; import org.apache.archiva.audit.AuditListener; import org.apache.archiva.audit.Auditable; @@ -28,6 +29,9 @@ import org.apache.archiva.common.utils.PathUtil; import org.apache.archiva.common.utils.VersionUtil; import org.apache.archiva.configuration.ArchivaConfiguration; import org.apache.archiva.configuration.RepositoryGroupConfiguration; +import org.apache.archiva.indexer.merger.IndexMerger; +import org.apache.archiva.indexer.merger.IndexMergerException; +import org.apache.archiva.indexer.search.RepositorySearch; import org.apache.archiva.model.ArchivaRepositoryMetadata; import org.apache.archiva.model.ArtifactReference; import org.apache.archiva.policies.ProxyDownloadException; @@ -93,7 +97,9 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * @@ -162,6 +168,12 @@ public class ArchivaDavResourceFactory @Named( value = "httpAuthenticator#basic" ) private HttpAuthenticator httpAuth; + @Inject + private IndexMerger indexMerger; + + @Inject + private RepositorySearch repositorySearch; + /** * Lock Manager - use simple implementation from JackRabbit */ @@ -1020,6 +1032,51 @@ public class ArchivaDavResourceFactory } } } + // remove last / + String pathInfo = StringUtils.removeEnd( request.getPathInfo(), "/" ); + if ( StringUtils.endsWith( path, ".indexer" ) ) + { + try + { + Set authzRepos = new HashSet(); + for ( String repository : repositories ) + { + try + { + if ( servletAuth.isAuthorized( activePrincipal, repository, + WebdavMethodUtil.getMethodPermission( + request.getMethod() ) ) ) + { + authzRepos.add( repository ); + authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) ); + } + } + catch ( UnauthorizedException e ) + { + // TODO: review exception handling + if ( log.isDebugEnabled() ) + { + log.debug( + "Skipping repository '" + repository + "' for user '" + activePrincipal + "': " + + e.getMessage() ); + } + } + } + + File mergedRepoDir = indexMerger.buildMergedIndex( authzRepos ); + mergedRepositoryContents.add( mergedRepoDir ); + + } + catch ( RepositoryAdminException e ) + { + throw new DavException( 500, e ); + } + catch ( IndexMergerException e ) + { + throw new DavException( 500, e ); + } + + } } else { diff --git a/pom.xml b/pom.xml index 45c395f22..c1ebe3fe9 100644 --- a/pom.xml +++ b/pom.xml @@ -470,7 +470,6 @@ - com.google.guava guava