Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[MRM-815] aggregate indices for repository groups. 

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1196840 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Olivier Lamy 2011-11-02 22:31:42 +00:00
parent b698377ca7
commit 0262091faa
11 changed files with 274 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
archiva-modules/archiva-base/archiva-indexer/pom.xml
View File

@ -45,6 +45,10 @@
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-digest</artifactId>

94
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java Normal file
View File

@ -0,0 +1,94 @@
package org.apache.archiva.indexer.merger;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import com.google.common.io.Files;
import com.google.inject.Inject;
import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
/**
* @author Olivier Lamy
* @since 1.4-M2
*/
@Service( "indexMerger#default" )
public class DefaultIndexMerger
implements IndexMerger
{
@Inject
private ManagedRepositoryAdmin managedRepositoryAdmin;
private MavenIndexerUtils mavenIndexerUtils;
private NexusIndexer indexer;
@javax.inject.Inject
public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils )
throws PlexusSisuBridgeException
{
this.indexer = plexusSisuBridge.lookup( NexusIndexer.class );
this.mavenIndexerUtils = mavenIndexerUtils;
}
public File buildMergedIndex( Collection<String> repositoriesIds )
throws IndexMergerException
{
File tempRepoFile = Files.createTempDir();
tempRepoFile.deleteOnExit();
String tempRepoId = tempRepoFile.getName();
try
{
IndexingContext indexingContext =
indexer.addIndexingContext( tempRepoId, tempRepoId, tempRepoFile, new File( tempRepoFile, ".indexer" ),
null, null, mavenIndexerUtils.getAllIndexCreators() );
for ( String repoId : repositoriesIds )
{
IndexingContext idxToMerge = indexer.getIndexingContexts().get( repoId );
if ( idxToMerge != null )
{
indexingContext.merge( idxToMerge.getIndexDirectory() );
}
}
return indexingContext.getIndexDirectoryFile();
}
catch ( IOException e )
{
throw new IndexMergerException( e.getMessage(), e );
}
catch ( UnsupportedExistingLuceneIndexException e )
{
throw new IndexMergerException( e.getMessage(), e );
}
}
}

37
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java Normal file
View File

@ -0,0 +1,37 @@
package org.apache.archiva.indexer.merger;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.io.File;
import java.util.Collection;
/**
* @author Olivier Lamy
* @since 1.4-M2
*/
public interface IndexMerger
{
/**
* @param repositoriesIds repositories Ids to merge content
* @return a temporary directory with a merge index (directory marked deleteOnExit)
* @throws IndexMergerException
*/
File buildMergedIndex( Collection<String> repositoriesIds )
throws IndexMergerException;
}

32
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerException.java Normal file
View File

@ -0,0 +1,32 @@
package org.apache.archiva.indexer.merger;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/**
* @author Olivier Lamy
* @since 1.4-M2
*/
public class IndexMergerException
extends Exception
{
public IndexMergerException( String message, Throwable t )
{
super( message, t );
}
}

8
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java
View File

@ -328,6 +328,10 @@ public class NexusRepositorySearch
if ( indexDir != null && !"".equals( indexDir ) )
{
indexDirectory = new File( repoConfig.getIndexDirectory() );
if ( !indexDirectory.isAbsolute() )
{
indexDirectory = new File( repoConfig.getLocation(), repoConfig.getIndexDirectory() );
}
}
else
{
@ -388,7 +392,7 @@ public class NexusRepositorySearch
}
private Set<String> getRemoteIndexingContextIds( String managedRepoId )
public Set<String> getRemoteIndexingContextIds( String managedRepoId )
throws RepositoryAdminException
{
Set<String> ids = new HashSet<String>();
@ -674,4 +678,6 @@ public class NexusRepositorySearch
return paginated;
}
}

6
archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/RepositorySearch.java
View File

@ -19,8 +19,11 @@ package org.apache.archiva.indexer.search;
* under the License.
*/
import org.apache.archiva.admin.model.RepositoryAdminException;
import java.util.Collection;
import java.util.List;
import java.util.Set;
public interface RepositorySearch
@ -52,4 +55,7 @@ public interface RepositorySearch
Collection<String> getAllGroupIds( String principal, List<String> selectedRepos )
throws RepositorySearchException;
Set<String> getRemoteIndexingContextIds( String managedRepoId )
throws RepositoryAdminException;
}

2
archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml
View File

@ -29,7 +29,7 @@
default-lazy-init="true">
<context:annotation-config/>
<context:component-scan base-package="org.apache.archiva.indexer.search"/>
<context:component-scan base-package="org.apache.archiva.indexer.search,org.apache.archiva.indexer.merger"/>
<bean id="logger" class="org.apache.archiva.common.utils.Slf4JPlexusLogger">
<constructor-arg type="java.lang.Class"><value>org.sonatype.nexus.index.DefaultNexusIndexer</value></constructor-arg>

41
archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java
View File

@ -47,7 +47,6 @@ import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.List;
@ -104,9 +103,6 @@ public class DefaultDownloadRemoteIndexScheduler
archivaConfiguration.addListener( this );
// TODO add indexContexts even if null
// FIXME get this from ArchivaAdministration
String appServerBase = System.getProperty( "appserver.base" );
nexusIndexer = plexusSisuBridge.lookup( NexusIndexer.class );
indexUpdater = plexusSisuBridge.lookup( IndexUpdater.class );
@ -114,29 +110,17 @@ public class DefaultDownloadRemoteIndexScheduler
for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() )
{
String contextKey = "remote-" + remoteRepository.getId();
if ( nexusIndexer.getIndexingContexts( ).get( contextKey ) != null )
IndexingContext context = nexusIndexer.getIndexingContexts().get( contextKey );
if ( context == null )
{
continue;
}
// create path
File repoDir = new File( appServerBase, "data/remotes/" + remoteRepository.getId( ) );
if ( !repoDir.exists( ) )
{
repoDir.mkdirs( );
}
File indexDirectory = new File( repoDir, ".index" );
if ( !indexDirectory.exists( ) )
{
indexDirectory.mkdirs( );
}
nexusIndexer.addIndexingContext( contextKey, remoteRepository.getId( ), repoDir, indexDirectory,
remoteRepository.getUrl( ), calculateIndexRemoteUrl( remoteRepository ),
mavenIndexerUtils.getAllIndexCreators( ) );
// TODO record jobs from configuration
if ( remoteRepository.isDownloadRemoteIndex() && StringUtils.isNotEmpty(
remoteRepository.getCronExpression() ) )
{
boolean fullDownload = indexDirectory.list( ).length == 0;
boolean fullDownload = context.getIndexDirectoryFile().list().length == 0;
scheduleDownloadRemote( remoteRepository.getId(), false, fullDownload );
}
}
@ -219,23 +203,6 @@ public class DefaultDownloadRemoteIndexScheduler
}
}
protected String calculateIndexRemoteUrl( RemoteRepository remoteRepository )
{
if ( StringUtils.startsWith( remoteRepository.getRemoteIndexUrl( ), "http" ) )
{
String baseUrl = remoteRepository.getRemoteIndexUrl( );
return baseUrl.endsWith( "/" ) ? StringUtils.substringBeforeLast( baseUrl, "/" ) : baseUrl;
}
String baseUrl = StringUtils.endsWith( remoteRepository.getUrl( ), "/" ) ? StringUtils.substringBeforeLast(
remoteRepository.getUrl( ), "/" ) : remoteRepository.getUrl( );
baseUrl = StringUtils.isEmpty( remoteRepository.getRemoteIndexUrl( ) )
? baseUrl + "/.index"
: baseUrl + "/" + remoteRepository.getRemoteIndexUrl( );
return baseUrl;
}
public TaskScheduler getTaskScheduler()
{
return taskScheduler;

4
archiva-modules/archiva-web/archiva-webdav/pom.xml
View File

@ -66,6 +66,10 @@
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-security</artifactId>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-indexer</artifactId>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>

57
archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java
View File

@ -19,6 +19,7 @@ package org.apache.archiva.webdav;
* under the License.
*/
import org.apache.archiva.admin.model.RepositoryAdminException;
import org.apache.archiva.audit.AuditEvent;
import org.apache.archiva.audit.AuditListener;
import org.apache.archiva.audit.Auditable;
@ -28,6 +29,9 @@ import org.apache.archiva.common.utils.PathUtil;
import org.apache.archiva.common.utils.VersionUtil;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.configuration.RepositoryGroupConfiguration;
import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.search.RepositorySearch;
import org.apache.archiva.model.ArchivaRepositoryMetadata;
import org.apache.archiva.model.ArtifactReference;
import org.apache.archiva.policies.ProxyDownloadException;
@ -93,7 +97,9 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
*
@ -162,6 +168,12 @@ public class ArchivaDavResourceFactory
@Named( value = "httpAuthenticator#basic" )
private HttpAuthenticator httpAuth;
@Inject
private IndexMerger indexMerger;
@Inject
private RepositorySearch repositorySearch;
/**
* Lock Manager - use simple implementation from JackRabbit
*/
@ -1020,6 +1032,51 @@ public class ArchivaDavResourceFactory
}
}
}
// remove last /
String pathInfo = StringUtils.removeEnd( request.getPathInfo(), "/" );
if ( StringUtils.endsWith( path, ".indexer" ) )
{
try
{
Set<String> authzRepos = new HashSet<String>();
for ( String repository : repositories )
{
try
{
if ( servletAuth.isAuthorized( activePrincipal, repository,
WebdavMethodUtil.getMethodPermission(
request.getMethod() ) ) )
{
authzRepos.add( repository );
authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) );
}
}
catch ( UnauthorizedException e )
{
// TODO: review exception handling
if ( log.isDebugEnabled() )
{
log.debug(
"Skipping repository '" + repository + "' for user '" + activePrincipal + "': "
+ e.getMessage() );
}
}
}
File mergedRepoDir = indexMerger.buildMergedIndex( authzRepos );
mergedRepositoryContents.add( mergedRepoDir );
}
catch ( RepositoryAdminException e )
{
throw new DavException( 500, e );
}
catch ( IndexMergerException e )
{
throw new DavException( 500, e );
}
}
}
else
{

1
pom.xml
View File

@ -470,7 +470,6 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>