[MRM-815] aggregate indices for repository groups.

git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1196840 13f79535-47bb-0310-9956-ffa450edef68

archiva-modules/archiva-base/archiva-indexer/pom.xml

@@ -45,6 +45,10 @@
       <groupId>org.codehaus.plexus</groupId>
       <artifactId>plexus-utils</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.codehaus.plexus</groupId>
       <artifactId>plexus-digest</artifactId>

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java

@@ -0,0 +1,94 @@
+package org.apache.archiva.indexer.merger;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.google.common.io.Files;
+import com.google.inject.Inject;
+import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
+import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
+import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
+import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
+import org.apache.maven.index.NexusIndexer;
+import org.apache.maven.index.context.IndexingContext;
+import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * @author Olivier Lamy
+ * @since 1.4-M2
+ */
+@Service( "indexMerger#default" )
+public class DefaultIndexMerger
+    implements IndexMerger
+{
+
+    @Inject
+    private ManagedRepositoryAdmin managedRepositoryAdmin;
+
+    private MavenIndexerUtils mavenIndexerUtils;
+
+    private NexusIndexer indexer;
+
+    @javax.inject.Inject
+    public DefaultIndexMerger( PlexusSisuBridge plexusSisuBridge, MavenIndexerUtils mavenIndexerUtils )
+        throws PlexusSisuBridgeException
+    {
+        this.indexer = plexusSisuBridge.lookup( NexusIndexer.class );
+        this.mavenIndexerUtils = mavenIndexerUtils;
+    }
+
+    public File buildMergedIndex( Collection<String> repositoriesIds )
+        throws IndexMergerException
+    {
+        File tempRepoFile = Files.createTempDir();
+        tempRepoFile.deleteOnExit();
+
+        String tempRepoId = tempRepoFile.getName();
+
+        try
+        {
+            IndexingContext indexingContext =
+                indexer.addIndexingContext( tempRepoId, tempRepoId, tempRepoFile, new File( tempRepoFile, ".indexer" ),
+                                            null, null, mavenIndexerUtils.getAllIndexCreators() );
+
+            for ( String repoId : repositoriesIds )
+            {
+                IndexingContext idxToMerge = indexer.getIndexingContexts().get( repoId );
+                if ( idxToMerge != null )
+                {
+                    indexingContext.merge( idxToMerge.getIndexDirectory() );
+                }
+            }
+
+            return indexingContext.getIndexDirectoryFile();
+        }
+        catch ( IOException e )
+        {
+            throw new IndexMergerException( e.getMessage(), e );
+        }
+        catch ( UnsupportedExistingLuceneIndexException e )
+        {
+            throw new IndexMergerException( e.getMessage(), e );
+        }
+    }
+}

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMerger.java

@@ -0,0 +1,37 @@
+package org.apache.archiva.indexer.merger;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.util.Collection;
+
+/**
+ * @author Olivier Lamy
+ * @since 1.4-M2
+ */
+public interface IndexMerger
+{
+    /**
+     * @param repositoriesIds repositories Ids to merge content
+     * @return a temporary directory with a merge index (directory marked deleteOnExit)
+     * @throws IndexMergerException
+     */
+    File buildMergedIndex( Collection<String> repositoriesIds )
+        throws IndexMergerException;
+}

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/merger/IndexMergerException.java

@@ -0,0 +1,32 @@
+package org.apache.archiva.indexer.merger;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * @author Olivier Lamy
+ * @since 1.4-M2
+ */
+public class IndexMergerException
+    extends Exception
+{
+    public IndexMergerException( String message, Throwable t )
+    {
+        super( message, t );
+    }
+}

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java

@@ -328,6 +328,10 @@ public class NexusRepositorySearch
                     if ( indexDir != null && !"".equals( indexDir ) )
                     {
                         indexDirectory = new File( repoConfig.getIndexDirectory() );
+                        if ( !indexDirectory.isAbsolute() )
+                        {
+                            indexDirectory = new File( repoConfig.getLocation(), repoConfig.getIndexDirectory() );
+                        }
                     }
                     else
                     {
@@ -388,7 +392,7 @@ public class NexusRepositorySearch
     }
 
 
-    private Set<String> getRemoteIndexingContextIds( String managedRepoId )
+    public Set<String> getRemoteIndexingContextIds( String managedRepoId )
         throws RepositoryAdminException
     {
         Set<String> ids = new HashSet<String>();
@@ -674,4 +678,6 @@ public class NexusRepositorySearch
 
         return paginated;
     }
+
+
 }

archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/RepositorySearch.java

@@ -19,29 +19,32 @@ package org.apache.archiva.indexer.search;
  * under the License.
  */
 
+import org.apache.archiva.admin.model.RepositoryAdminException;
+
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 
 public interface RepositorySearch
 {
     /**
      * Quick search.
-     * 
+     *
      * @param principal
      * @param selectedRepos
      * @param term
      * @param limits
-     * @param previousSearchTerms 
+     * @param previousSearchTerms
      * @return
      */
     SearchResults search( String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
                           List<String> previousSearchTerms )
         throws RepositorySearchException;
-    
+
     /**
      * Advanced search.
-     * 
+     *
      * @param principal
      * @param searchFields
      * @param limits
@@ -49,7 +52,10 @@ public interface RepositorySearch
      */
     SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits )
         throws RepositorySearchException;
-    
+
     Collection<String> getAllGroupIds( String principal, List<String> selectedRepos )
         throws RepositorySearchException;
+
+    Set<String> getRemoteIndexingContextIds( String managedRepoId )
+        throws RepositoryAdminException;
 }

archiva-modules/archiva-base/archiva-indexer/src/main/resources/META-INF/spring-context.xml

@@ -29,7 +29,7 @@
        default-lazy-init="true">
 
   <context:annotation-config/>
-  <context:component-scan base-package="org.apache.archiva.indexer.search"/>
+  <context:component-scan base-package="org.apache.archiva.indexer.search,org.apache.archiva.indexer.merger"/>
 
   <bean id="logger" class="org.apache.archiva.common.utils.Slf4JPlexusLogger">
     <constructor-arg type="java.lang.Class"><value>org.sonatype.nexus.index.DefaultNexusIndexer</value></constructor-arg>

archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/DefaultDownloadRemoteIndexScheduler.java

@@ -47,7 +47,6 @@ import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import javax.inject.Inject;
 import javax.inject.Named;
-import java.io.File;
 import java.io.IOException;
 import java.util.Date;
 import java.util.List;
@@ -62,7 +61,7 @@ public class DefaultDownloadRemoteIndexScheduler
     implements ConfigurationListener, DownloadRemoteIndexScheduler
 {
 
-    private Logger log = LoggerFactory.getLogger( getClass( ) );
+    private Logger log = LoggerFactory.getLogger( getClass() );
 
     @Inject
     @Named( value = "taskScheduler#indexDownloadRemote" )
@@ -94,50 +93,35 @@ public class DefaultDownloadRemoteIndexScheduler
     private IndexUpdater indexUpdater;
 
     // store ids about currently running remote download : updated in DownloadRemoteIndexTask
-    private List<String> runningRemoteDownloadIds = new CopyOnWriteArrayList<String>( );
+    private List<String> runningRemoteDownloadIds = new CopyOnWriteArrayList<String>();
 
     @PostConstruct
-    public void startup( )
+    public void startup()
         throws ArchivaException, RepositoryAdminException, PlexusSisuBridgeException, IOException,
         UnsupportedExistingLuceneIndexException, DownloadRemoteIndexException
     {
         archivaConfiguration.addListener( this );
         // TODO add indexContexts even if null
 
-        // FIXME get this from ArchivaAdministration
-        String appServerBase = System.getProperty( "appserver.base" );
-
         nexusIndexer = plexusSisuBridge.lookup( NexusIndexer.class );
 
         indexUpdater = plexusSisuBridge.lookup( IndexUpdater.class );
 
-        for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories( ) )
+        for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() )
         {
-            String contextKey = "remote-" + remoteRepository.getId( );
+            String contextKey = "remote-" + remoteRepository.getId();
-            if ( nexusIndexer.getIndexingContexts( ).get( contextKey ) != null )
+            IndexingContext context = nexusIndexer.getIndexingContexts().get( contextKey );
+            if ( context == null )
             {
                 continue;
             }
-            // create path
+
-            File repoDir = new File( appServerBase, "data/remotes/" + remoteRepository.getId( ) );
-            if ( !repoDir.exists( ) )
-            {
-                repoDir.mkdirs( );
-            }
-            File indexDirectory = new File( repoDir, ".index" );
-            if ( !indexDirectory.exists( ) )
-            {
-                indexDirectory.mkdirs( );
-            }
-            nexusIndexer.addIndexingContext( contextKey, remoteRepository.getId( ), repoDir, indexDirectory,
-                                             remoteRepository.getUrl( ), calculateIndexRemoteUrl( remoteRepository ),
-                                             mavenIndexerUtils.getAllIndexCreators( ) );
             // TODO record jobs from configuration
-            if ( remoteRepository.isDownloadRemoteIndex( ) && StringUtils.isNotEmpty(
+            if ( remoteRepository.isDownloadRemoteIndex() && StringUtils.isNotEmpty(
-                remoteRepository.getCronExpression( ) ) )
+                remoteRepository.getCronExpression() ) )
             {
-                boolean fullDownload = indexDirectory.list( ).length == 0;
+                boolean fullDownload = context.getIndexDirectoryFile().list().length == 0;
-                scheduleDownloadRemote( remoteRepository.getId( ), false, fullDownload );
+                scheduleDownloadRemote( remoteRepository.getId(), false, fullDownload );
             }
         }
 
@@ -145,13 +129,13 @@ public class DefaultDownloadRemoteIndexScheduler
     }
 
     @PreDestroy
-    public void shutdown( )
+    public void shutdown()
         throws RepositoryAdminException, IOException
     {
-        for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories( ) )
+        for ( RemoteRepository remoteRepository : remoteRepositoryAdmin.getRemoteRepositories() )
         {
-            String contextKey = "remote-" + remoteRepository.getId( );
+            String contextKey = "remote-" + remoteRepository.getId();
-            IndexingContext context = nexusIndexer.getIndexingContexts( ).get( contextKey );
+            IndexingContext context = nexusIndexer.getIndexingContexts().get( contextKey );
             if ( context == null )
             {
                 continue;
@@ -178,65 +162,48 @@ public class DefaultDownloadRemoteIndexScheduler
                 return;
             }
             NetworkProxy networkProxy = null;
-            if ( StringUtils.isNotBlank( remoteRepository.getRemoteDownloadNetworkProxyId( ) ) )
+            if ( StringUtils.isNotBlank( remoteRepository.getRemoteDownloadNetworkProxyId() ) )
             {
-                networkProxy = networkProxyAdmin.getNetworkProxy( remoteRepository.getRemoteDownloadNetworkProxyId( ) );
+                networkProxy = networkProxyAdmin.getNetworkProxy( remoteRepository.getRemoteDownloadNetworkProxyId() );
                 if ( networkProxy == null )
                 {
                     log.warn(
                         "your remote repository is configured to download remote index trought a proxy we cannot find id:{}",
-                        remoteRepository.getRemoteDownloadNetworkProxyId( ) );
+                        remoteRepository.getRemoteDownloadNetworkProxyId() );
                 }
             }
 
             DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest =
-                new DownloadRemoteIndexTaskRequest( ).setRemoteRepository( remoteRepository ).setNetworkProxy(
+                new DownloadRemoteIndexTaskRequest().setRemoteRepository( remoteRepository ).setNetworkProxy(
                     networkProxy ).setFullDownload( fullDownload ).setWagonFactory( wagonFactory ).setNexusIndexer(
                     nexusIndexer ).setIndexUpdater( indexUpdater );
 
             if ( now )
             {
-                log.info( "schedule download remote index for repository {}", remoteRepository.getId( ) );
+                log.info( "schedule download remote index for repository {}", remoteRepository.getId() );
                 // do it in async
                 taskScheduler.schedule(
                     new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ),
-                    new Date( ) );
+                    new Date() );
             }
             else
             {
                 log.info( "schedule download remote index for repository {} with cron expression {}",
-                          remoteRepository.getId( ), remoteRepository.getCronExpression( ) );
+                          remoteRepository.getId(), remoteRepository.getCronExpression() );
                 taskScheduler.schedule(
                     new DownloadRemoteIndexTask( downloadRemoteIndexTaskRequest, this.runningRemoteDownloadIds ),
-                    new CronTrigger( remoteRepository.getCronExpression( ) ) );
+                    new CronTrigger( remoteRepository.getCronExpression() ) );
             }
 
         }
         catch ( RepositoryAdminException e )
         {
-            log.error( e.getMessage( ), e );
+            log.error( e.getMessage(), e );
-            throw new DownloadRemoteIndexException( e.getMessage( ), e );
+            throw new DownloadRemoteIndexException( e.getMessage(), e );
         }
     }
 
-    protected String calculateIndexRemoteUrl( RemoteRepository remoteRepository )
+    public TaskScheduler getTaskScheduler()
-    {
-        if ( StringUtils.startsWith( remoteRepository.getRemoteIndexUrl( ), "http" ) )
-        {
-            String baseUrl = remoteRepository.getRemoteIndexUrl( );
-            return baseUrl.endsWith( "/" ) ? StringUtils.substringBeforeLast( baseUrl, "/" ) : baseUrl;
-        }
-        String baseUrl = StringUtils.endsWith( remoteRepository.getUrl( ), "/" ) ? StringUtils.substringBeforeLast(
-            remoteRepository.getUrl( ), "/" ) : remoteRepository.getUrl( );
-
-        baseUrl = StringUtils.isEmpty( remoteRepository.getRemoteIndexUrl( ) )
-            ? baseUrl + "/.index"
-            : baseUrl + "/" + remoteRepository.getRemoteIndexUrl( );
-        return baseUrl;
-
-    }
-
-    public TaskScheduler getTaskScheduler( )
     {
         return taskScheduler;
     }

archiva-modules/archiva-web/archiva-webdav/pom.xml

@@ -66,6 +66,10 @@
       <groupId>org.apache.archiva</groupId>
       <artifactId>archiva-security</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.archiva</groupId>
+      <artifactId>archiva-indexer</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.jackrabbit</groupId>
       <artifactId>jackrabbit-webdav</artifactId>

archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java

@@ -19,6 +19,7 @@ package org.apache.archiva.webdav;
  * under the License.
  */
 
+import org.apache.archiva.admin.model.RepositoryAdminException;
 import org.apache.archiva.audit.AuditEvent;
 import org.apache.archiva.audit.AuditListener;
 import org.apache.archiva.audit.Auditable;
@@ -28,6 +29,9 @@ import org.apache.archiva.common.utils.PathUtil;
 import org.apache.archiva.common.utils.VersionUtil;
 import org.apache.archiva.configuration.ArchivaConfiguration;
 import org.apache.archiva.configuration.RepositoryGroupConfiguration;
+import org.apache.archiva.indexer.merger.IndexMerger;
+import org.apache.archiva.indexer.merger.IndexMergerException;
+import org.apache.archiva.indexer.search.RepositorySearch;
 import org.apache.archiva.model.ArchivaRepositoryMetadata;
 import org.apache.archiva.model.ArtifactReference;
 import org.apache.archiva.policies.ProxyDownloadException;
@@ -93,7 +97,9 @@ import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  *
@@ -162,6 +168,12 @@ public class ArchivaDavResourceFactory
     @Named( value = "httpAuthenticator#basic" )
     private HttpAuthenticator httpAuth;
 
+    @Inject
+    private IndexMerger indexMerger;
+
+    @Inject
+    private RepositorySearch repositorySearch;
+
     /**
      * Lock Manager - use simple implementation from JackRabbit
      */
@@ -1020,6 +1032,51 @@ public class ArchivaDavResourceFactory
                     }
                 }
             }
+            // remove last /
+            String pathInfo = StringUtils.removeEnd( request.getPathInfo(), "/"  );
+            if ( StringUtils.endsWith( path, ".indexer" ) )
+            {
+                try
+                {
+                    Set<String> authzRepos = new HashSet<String>();
+                    for ( String repository : repositories )
+                    {
+                        try
+                        {
+                            if ( servletAuth.isAuthorized( activePrincipal, repository,
+                                                           WebdavMethodUtil.getMethodPermission(
+                                                               request.getMethod() ) ) )
+                            {
+                                authzRepos.add( repository );
+                                authzRepos.addAll( this.repositorySearch.getRemoteIndexingContextIds( repository ) );
+                            }
+                        }
+                        catch ( UnauthorizedException e )
+                        {
+                            // TODO: review exception handling
+                            if ( log.isDebugEnabled() )
+                            {
+                                log.debug(
+                                    "Skipping repository '" + repository + "' for user '" + activePrincipal + "': "
+                                        + e.getMessage() );
+                            }
+                        }
+                    }
+
+                    File mergedRepoDir = indexMerger.buildMergedIndex( authzRepos );
+                    mergedRepositoryContents.add( mergedRepoDir );
+
+                }
+                catch ( RepositoryAdminException e )
+                {
+                    throw new DavException( 500, e );
+                }
+                catch ( IndexMergerException e )
+                {
+                    throw new DavException( 500, e );
+                }
+
+            }
         }
         else
         {

pom.xml

@@ -470,7 +470,6 @@
         </exclusions>
       </dependency>
 
-
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>