[MRM-463] Metadata merging doesn't work.

Adding new Metadata Updater Consumer.
Updating archiva-configuration default for new consumer.
Updating archiva-proxy for new Metadata Tools approach.

git-svn-id: https://svn.apache.org/repos/asf/maven/archiva/trunk@574121 13f79535-47bb-0310-9956-ffa450edef68

archiva-base/archiva-configuration/src/main/resources/org/apache/maven/archiva/configuration/default-archiva.xml

@@ -138,6 +138,7 @@
       <knownContentConsumer>index-content</knownContentConsumer>
       <knownContentConsumer>auto-remove</knownContentConsumer>
       <knownContentConsumer>auto-rename</knownContentConsumer>
+      <knownContentConsumer>metadata-updater</knownContentConsumer>
       <!--knownContentConsumer>repository-purge</knownContentConsumer-->
     </knownContentConsumers>
     <invalidContentConsumers>

archiva-base/archiva-configuration/src/test/conf/autodetect-v1.xml

@@ -156,6 +156,7 @@
       <knownContentConsumer>index-content</knownContentConsumer>
       <knownContentConsumer>auto-remove</knownContentConsumer>
       <knownContentConsumer>auto-rename</knownContentConsumer>
+      <knownContentConsumer>metadata-updater</knownContentConsumer>
     </knownContentConsumers>
     <invalidContentConsumers>
       <invalidContentConsumer>update-db-bad-content</invalidContentConsumer>

archiva-base/archiva-configuration/src/test/conf/repository-manager.xml

@@ -157,6 +157,7 @@
       <knownContentConsumer>index-content</knownContentConsumer>
       <knownContentConsumer>auto-remove</knownContentConsumer>
       <knownContentConsumer>auto-rename</knownContentConsumer>
+      <knownContentConsumer>metadata-updater</knownContentConsumer>
     </knownContentConsumers>
     <invalidContentConsumers>
       <invalidContentConsumer>update-db-bad-content</invalidContentConsumer>

archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java

@@ -89,6 +89,9 @@ public void testGetConfigurationFromDefaults()
         assertTrue( "check managed repositories", repository.isIndexed() );
     }
 
+    /**
+     * Ensures that the provided configuration matches the details present in the archiva-default.xml file.
+     */
     private void assertConfiguration( Configuration configuration )
         throws Exception
     {
@@ -100,7 +103,7 @@ private void assertConfiguration( Configuration configuration )
         RepositoryScanningConfiguration repoScanning = configuration.getRepositoryScanning();
         assertNotNull( "check repository scanning", repoScanning );
         assertEquals( "check file types", 4, repoScanning.getFileTypes().size() );
-        assertEquals( "check known consumers", 8, repoScanning.getKnownContentConsumers().size() );
+        assertEquals( "check known consumers", 9, repoScanning.getKnownContentConsumers().size() );
         assertEquals( "check invalid consumers", 1, repoScanning.getInvalidContentConsumers().size() );
 
         List patterns = filetypes.getFileTypePatterns( "artifacts" );

archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java

@@ -75,7 +75,7 @@ public class ArtifactMissingChecksumsConsumer extends AbstractMonitoredConsumer
     /**
      * @plexus.requirement role="org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayout"
      */
-    private Map bidirectionalLayoutMap;
+    private Map bidirectionalLayoutMap; // TODO: replace with new bidir-repo-layout-factory
 
     /**
      * @plexus.requirement role-hint="sha1"
@@ -137,7 +137,7 @@ public void beginScan( ArchivaRepository repository ) throws ConsumerException
         if ( !bidirectionalLayoutMap.containsKey( layoutName ) )
         {
             throw new ConsumerException( "Unable to process repository with layout [" + layoutName
-                            + "] as there is no coresponding " + BidirectionalRepositoryLayout.class.getName()
+                            + "] as there is no corresponding " + BidirectionalRepositoryLayout.class.getName()
                             + " implementation available." );
         }
 

archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java

@@ -0,0 +1,287 @@
+package org.apache.maven.archiva.consumers.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.archiva.configuration.ArchivaConfiguration;
+import org.apache.maven.archiva.configuration.ConfigurationNames;
+import org.apache.maven.archiva.configuration.FileTypes;
+import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer;
+import org.apache.maven.archiva.consumers.ConsumerException;
+import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer;
+import org.apache.maven.archiva.model.ArchivaRepository;
+import org.apache.maven.archiva.model.ArtifactReference;
+import org.apache.maven.archiva.model.ProjectReference;
+import org.apache.maven.archiva.model.VersionedReference;
+import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayout;
+import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayoutFactory;
+import org.apache.maven.archiva.repository.layout.LayoutException;
+import org.apache.maven.archiva.repository.metadata.MetadataTools;
+import org.apache.maven.archiva.repository.metadata.RepositoryMetadataException;
+import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable;
+import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException;
+import org.codehaus.plexus.registry.Registry;
+import org.codehaus.plexus.registry.RegistryListener;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * MetadataUpdaterConsumer will create and update the metadata present within the repository.
+ *
+ * @author <a href="mailto:joakime@apache.org">Joakim Erdfelt</a>
+ * @version $Id$
+
+ * @plexus.component role="org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer"
+ *                   role-hint="metadata-updater"
+ *                   instantiation-strategy="per-lookup"
+ */
+public class MetadataUpdaterConsumer
+    extends AbstractMonitoredConsumer
+    implements KnownRepositoryContentConsumer, RegistryListener, Initializable
+{
+    /**
+     * @plexus.configuration default-value="metadata-updater"
+     */
+    private String id;
+
+    /**
+     * @plexus.configuration default-value="Update / Create maven-metadata.xml files"
+     */
+    private String description;
+
+    /**
+     * @plexus.requirement
+     */
+    private BidirectionalRepositoryLayoutFactory layoutFactory;
+
+    /**
+     * @plexus.requirement
+     */
+    private MetadataTools metadataTools;
+
+    /**
+     * @plexus.requirement
+     */
+    private ArchivaConfiguration configuration;
+
+    /**
+     * @plexus.requirement
+     */
+    private FileTypes filetypes;
+
+    private static final String TYPE_METADATA_BAD_INTERNAL_REF = "metadata-bad-internal-ref";
+
+    private static final String TYPE_METADATA_WRITE_FAILURE = "metadata-write-failure";
+
+    private static final String TYPE_METADATA_IO = "metadata-io-warning";
+
+    private ArchivaRepository repository;
+
+    private File repositoryDir;
+
+    private BidirectionalRepositoryLayout repositoryLayout;
+
+    private List includes = new ArrayList();
+
+    private long scanStartTimestamp = 0;
+
+    public String getDescription()
+    {
+        return description;
+    }
+
+    public String getId()
+    {
+        return id;
+    }
+
+    public void setIncludes( List includes )
+    {
+        this.includes = includes;
+    }
+
+    public void beginScan( ArchivaRepository repository )
+        throws ConsumerException
+    {
+        if ( !repository.isManaged() )
+        {
+            throw new ConsumerException( "Consumer requires managed repository." );
+        }
+
+        this.repository = repository;
+        this.repositoryDir = new File( repository.getUrl().getPath() );
+        try
+        {
+            this.repositoryLayout = layoutFactory.getLayout( repository.getLayoutType() );
+        }
+        catch ( LayoutException e )
+        {
+            throw new ConsumerException( "Cannot operate with bad layout definition on repo [" + repository.getId()
+                + "]: " + e.getMessage(), e );
+        }
+        this.scanStartTimestamp = System.currentTimeMillis();
+    }
+
+    public void completeScan()
+    {
+        /* do nothing here */
+    }
+
+    public List getExcludes()
+    {
+        return null;
+    }
+
+    public List getIncludes()
+    {
+        return this.includes;
+    }
+
+    public void processFile( String path )
+        throws ConsumerException
+    {
+        try
+        {
+            ArtifactReference artifact = this.repositoryLayout.toArtifactReference( path );
+            updateVersionMetadata( artifact, path );
+            updateProjectMetadata( artifact, path );
+        }
+        catch ( LayoutException e )
+        {
+            throw new ConsumerException( "Unable to convert to artifact reference: " + path, e );
+        }
+    }
+
+    private void updateProjectMetadata( ArtifactReference artifact, String path )
+    {
+        ProjectReference projectRef = new ProjectReference();
+        projectRef.setGroupId( artifact.getGroupId() );
+        projectRef.setArtifactId( artifact.getArtifactId() );
+
+        try
+        {
+            String metadataPath = this.metadataTools.toPath( projectRef );
+
+            File projectMetadata = new File( this.repositoryDir, metadataPath );
+
+            if ( projectMetadata.exists() && ( projectMetadata.lastModified() >= this.scanStartTimestamp ) )
+            {
+                // This metadata is up to date. skip it.
+                getLogger().debug( "Skipping uptodate metadata: " + this.metadataTools.toPath( projectRef ) );
+                return;
+            }
+
+            metadataTools.updateMetadata( this.repository, projectRef );
+            getLogger().info( "Updated metadata: " + this.metadataTools.toPath( projectRef ) );
+        }
+        catch ( LayoutException e )
+        {
+            triggerConsumerWarning( TYPE_METADATA_BAD_INTERNAL_REF, "Unable to convert path [" + path
+                + "] to an internal project reference: " + e.getMessage() );
+        }
+        catch ( RepositoryMetadataException e )
+        {
+            triggerConsumerError( TYPE_METADATA_WRITE_FAILURE, "Unable to write project metadata for artifact [" + path
+                + "]: " + e.getMessage() );
+        }
+        catch ( IOException e )
+        {
+            triggerConsumerWarning( TYPE_METADATA_IO, "Project metadata not written due to IO warning: "
+                + e.getMessage() );
+        }
+    }
+
+    private void updateVersionMetadata( ArtifactReference artifact, String path )
+    {
+        VersionedReference versionRef = new VersionedReference();
+        versionRef.setGroupId( artifact.getGroupId() );
+        versionRef.setArtifactId( artifact.getArtifactId() );
+        versionRef.setVersion( artifact.getVersion() );
+
+        try
+        {
+            String metadataPath = this.metadataTools.toPath( versionRef );
+
+            File projectMetadata = new File( this.repositoryDir, metadataPath );
+
+            if ( projectMetadata.exists() && ( projectMetadata.lastModified() >= this.scanStartTimestamp ) )
+            {
+                // This metadata is up to date. skip it.
+                getLogger().debug( "Skipping uptodate metadata: " + this.metadataTools.toPath( versionRef ) );
+                return;
+            }
+
+            metadataTools.updateMetadata( this.repository, versionRef );
+            getLogger().info( "Updated metadata: " + this.metadataTools.toPath( versionRef ) );
+        }
+        catch ( LayoutException e )
+        {
+            triggerConsumerWarning( TYPE_METADATA_BAD_INTERNAL_REF, "Unable to convert path [" + path
+                + "] to an internal version reference: " + e.getMessage() );
+        }
+        catch ( RepositoryMetadataException e )
+        {
+            triggerConsumerError( TYPE_METADATA_WRITE_FAILURE, "Unable to write version metadata for artifact [" + path
+                + "]: " + e.getMessage() );
+        }
+        catch ( IOException e )
+        {
+            triggerConsumerWarning( TYPE_METADATA_IO, "Version metadata not written due to IO warning: "
+                + e.getMessage() );
+        }
+    }
+
+    public boolean isPermanent()
+    {
+        return false;
+    }
+
+    public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
+    {
+        getLogger().info( "After Configuration Change: propertyName [" + propertyName + "]" );
+
+        if ( ConfigurationNames.isRepositoryScanning( propertyName ) )
+        {
+            initIncludes();
+        }
+    }
+
+    public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
+    {
+        /* do nothing here */
+    }
+
+    private void initIncludes()
+    {
+        includes.clear();
+
+        includes.addAll( filetypes.getFileTypePatterns( FileTypes.ARTIFACTS ) );
+    }
+
+    public void initialize()
+        throws InitializationException
+    {
+        configuration.addChangeListener( this );
+
+        initIncludes();
+    }
+}

archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java

@@ -174,7 +174,7 @@ public File fetchFromProxies( ArchivaRepository repository, VersionedReference m
         {
             ProxyConnector connector = (ProxyConnector) it.next();
             ArchivaRepository targetRepository = connector.getTargetRepository();
-            String targetPath = getLayout( targetRepository ).toPath( metadata );
+            String targetPath = metadataTools.toPath( metadata );
             
             File localRepoFile = toLocalRepoFile( repository, targetRepository, targetPath );
             File downloadedFile = transferFile( connector, targetRepository, targetPath, localRepoFile, requestProperties );
@@ -240,7 +240,7 @@ public File fetchFromProxies( ArchivaRepository repository, ProjectReference met
         {
             ProxyConnector connector = (ProxyConnector) it.next();
             ArchivaRepository targetRepository = connector.getTargetRepository();
-            String targetPath = getLayout( targetRepository ).toPath( metadata );
+            String targetPath = metadataTools.toPath( metadata );
 
             File localRepoFile = toLocalRepoFile( repository, targetRepository, targetPath );
             File downloadedFile = transferFile( connector, targetRepository, targetPath, localRepoFile, requestProperties );
@@ -328,16 +328,14 @@ private File toLocalFile( ArchivaRepository repository, ArtifactReference artifa
     private File toLocalFile( ArchivaRepository repository, ProjectReference metadata )
         throws ProxyException
     {
-        BidirectionalRepositoryLayout sourceLayout = getLayout( repository );
+        String sourcePath = metadataTools.toPath( metadata );
-        String sourcePath = sourceLayout.toPath( metadata );
         return new File( repository.getUrl().getPath(), sourcePath );
     }
 
     private File toLocalFile( ArchivaRepository repository, VersionedReference metadata )
         throws ProxyException
     {
-        BidirectionalRepositoryLayout sourceLayout = getLayout( repository );
+        String sourcePath = metadataTools.toPath( metadata );
-        String sourcePath = sourceLayout.toPath( metadata );
         return new File( repository.getUrl().getPath(), sourcePath );
     }
 

archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/AbstractProxyTestCase.java

@@ -27,8 +27,6 @@
 import org.apache.maven.archiva.model.ArchivaArtifact;
 import org.apache.maven.archiva.model.ArchivaRepository;
 import org.apache.maven.archiva.model.ArtifactReference;
-import org.apache.maven.archiva.model.ProjectReference;
-import org.apache.maven.archiva.model.VersionedReference;
 import org.apache.maven.archiva.policies.urlcache.UrlFailureCache;
 import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayout;
 import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayoutFactory;
@@ -256,22 +254,6 @@ protected ArchivaRepository createManagedLegacyRepository()
                                  "Test Managed (Legacy) Repository", "legacy" );
     }
 
-    protected ProjectReference createProjectReference( String layoutType, String path )
-        throws Exception
-    {
-        BidirectionalRepositoryLayout layout = layoutFactory.getLayout( layoutType );
-        ProjectReference metadata = layout.toProjectReference( path );
-        return metadata;
-    }
-
-    protected VersionedReference createVersionedReference( String layoutType, String path )
-        throws Exception
-    {
-        BidirectionalRepositoryLayout layout = layoutFactory.getLayout( layoutType );
-        VersionedReference metadata = layout.toVersionedReference( path );
-        return metadata;
-    }
-
     protected ArchivaRepository createProxiedLegacyRepository()
     {
         return createRepository( "src/test/repositories/legacy-proxied", "testProxiedLegacyRepo",

archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/MetadataTransferTest.java

@@ -677,7 +677,7 @@ private void assertFetchProject( String requestedResource )
     {
         File expectedFile = new File( managedDefaultDir, requestedResource );
 
-        ProjectReference metadata = createProjectReference( "default", requestedResource );
+        ProjectReference metadata = createProjectReference( requestedResource );
 
         File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata );
 
@@ -685,6 +685,12 @@ private void assertFetchProject( String requestedResource )
         assertNoTempFiles( expectedFile );
     }
 
+    private ProjectReference createProjectReference( String path )
+        throws RepositoryMetadataException
+    {
+        return metadataTools.toProjectReference( path );
+    }
+
     /**
      * Transfer the metadata file, not expected to succeed.
      * 
@@ -695,7 +701,7 @@ private void assertFetchProjectFailed( String requestedResource )
         throws Exception
     {
         File expectedFile = new File( managedDefaultDir, requestedResource );
-        ProjectReference metadata = createProjectReference( "default", requestedResource );
+        ProjectReference metadata = createProjectReference( requestedResource );
 
         File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata );
 
@@ -714,7 +720,7 @@ private void assertFetchVersioned( String requestedResource )
     {
         File expectedFile = new File( managedDefaultDir, requestedResource );
 
-        VersionedReference metadata = createVersionedReference( "default", requestedResource );
+        VersionedReference metadata = createVersionedReference( requestedResource );
 
         File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata );
 
@@ -722,6 +728,12 @@ private void assertFetchVersioned( String requestedResource )
         assertNoTempFiles( expectedFile );
     }
 
+    private VersionedReference createVersionedReference( String path )
+        throws RepositoryMetadataException
+    {
+        return metadataTools.toVersionedReference( path );
+    }
+
     /**
      * Transfer the metadata file, not expected to succeed.
      * 
@@ -732,7 +744,7 @@ private void assertFetchVersionedFailed( String requestedResource )
         throws Exception
     {
         File expectedFile = new File( managedDefaultDir, requestedResource );
-        VersionedReference metadata = createVersionedReference( "default", requestedResource );
+        VersionedReference metadata = createVersionedReference( requestedResource );
 
         File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata );
 
@@ -813,7 +825,7 @@ private void assertProjectMetadataContents( String requestedResource, String exp
         File actualFile = new File( managedDefaultDir, requestedResource );
         assertTrue( actualFile.exists() );
 
-        ProjectReference metadata = createProjectReference( "default", requestedResource );
+        ProjectReference metadata = createProjectReference( requestedResource );
 
         // Build expected metadata XML
         StringWriter expectedMetadataXml = new StringWriter();
@@ -846,7 +858,7 @@ private void assertReleaseMetadataContents( String requestedResource )
         File actualFile = new File( managedDefaultDir, requestedResource );
         assertTrue( "Release Metadata should exist: " + requestedResource, actualFile.exists() );
 
-        VersionedReference metadata = createVersionedReference( "default", requestedResource );
+        VersionedReference metadata = createVersionedReference( requestedResource );
 
         // Build expected metadata XML
         StringWriter expectedMetadataXml = new StringWriter();
@@ -879,7 +891,7 @@ private void assertSnapshotMetadataContents( String requestedResource, String ex
         File actualFile = new File( managedDefaultDir, requestedResource );
         assertTrue( "Snapshot Metadata should exist: " + requestedResource, actualFile.exists() );
 
-        VersionedReference actualMetadata = createVersionedReference( "default", requestedResource );
+        VersionedReference actualMetadata = createVersionedReference( requestedResource );
 
         assertSnapshotMetadata( actualFile, actualMetadata, expectedDate, expectedTime, expectedBuildnumber );
     }
@@ -905,7 +917,7 @@ private void assertRepoSnapshotMetadataContents( String proxiedRepoId, String re
         File actualFile = new File( managedDefaultDir, proxiedFile );
         assertTrue( "Repo Specific Snapshot Metadata should exist: " + requestedResource, actualFile.exists() );
 
-        VersionedReference actualMetadata = createVersionedReference( "default", requestedResource );
+        VersionedReference actualMetadata = createVersionedReference( requestedResource );
 
         assertSnapshotMetadata( actualFile, actualMetadata, expectedDate, expectedTime, expectedBuildnumber );
     }
@@ -955,7 +967,7 @@ private void assertRepoProjectMetadata( String proxiedRepoId, String requestedRe
         File actualFile = new File( managedDefaultDir, proxiedFile );
         assertTrue( actualFile.exists() );
 
-        ProjectReference metadata = createProjectReference( "default", requestedResource );
+        ProjectReference metadata = createProjectReference( requestedResource );
 
         // Build expected metadata XML
         StringWriter expectedMetadataXml = new StringWriter();
@@ -990,7 +1002,7 @@ private void assertRepoReleaseMetadataContents( String proxiedRepoId, String req
         File actualFile = new File( managedDefaultDir, proxiedFile );
         assertTrue( "Release metadata for repo should exist: " + actualFile, actualFile.exists() );
 
-        VersionedReference metadata = createVersionedReference( "default", requestedResource );
+        VersionedReference metadata = createVersionedReference( requestedResource );
 
         // Build expected metadata XML
         StringWriter expectedMetadataXml = new StringWriter();