diff --git a/archiva-base/archiva-configuration/src/main/resources/org/apache/maven/archiva/configuration/default-archiva.xml b/archiva-base/archiva-configuration/src/main/resources/org/apache/maven/archiva/configuration/default-archiva.xml index 6e7d4681b..8fd0669a5 100644 --- a/archiva-base/archiva-configuration/src/main/resources/org/apache/maven/archiva/configuration/default-archiva.xml +++ b/archiva-base/archiva-configuration/src/main/resources/org/apache/maven/archiva/configuration/default-archiva.xml @@ -138,6 +138,7 @@ index-content auto-remove auto-rename + metadata-updater diff --git a/archiva-base/archiva-configuration/src/test/conf/autodetect-v1.xml b/archiva-base/archiva-configuration/src/test/conf/autodetect-v1.xml index 5f35f5d37..7a50c699e 100644 --- a/archiva-base/archiva-configuration/src/test/conf/autodetect-v1.xml +++ b/archiva-base/archiva-configuration/src/test/conf/autodetect-v1.xml @@ -156,6 +156,7 @@ index-content auto-remove auto-rename + metadata-updater update-db-bad-content diff --git a/archiva-base/archiva-configuration/src/test/conf/repository-manager.xml b/archiva-base/archiva-configuration/src/test/conf/repository-manager.xml index 01eacf5af..f5dd079f5 100644 --- a/archiva-base/archiva-configuration/src/test/conf/repository-manager.xml +++ b/archiva-base/archiva-configuration/src/test/conf/repository-manager.xml @@ -157,6 +157,7 @@ index-content auto-remove auto-rename + metadata-updater update-db-bad-content diff --git a/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java b/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java index 1ed76eb63..d37ae51ef 100644 --- a/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java +++ b/archiva-base/archiva-configuration/src/test/java/org/apache/maven/archiva/configuration/ArchivaConfigurationTest.java @@ -89,6 +89,9 @@ public void testGetConfigurationFromDefaults() assertTrue( "check managed repositories", repository.isIndexed() ); } + /** + * Ensures that the provided configuration matches the details present in the archiva-default.xml file. + */ private void assertConfiguration( Configuration configuration ) throws Exception { @@ -100,7 +103,7 @@ private void assertConfiguration( Configuration configuration ) RepositoryScanningConfiguration repoScanning = configuration.getRepositoryScanning(); assertNotNull( "check repository scanning", repoScanning ); assertEquals( "check file types", 4, repoScanning.getFileTypes().size() ); - assertEquals( "check known consumers", 8, repoScanning.getKnownContentConsumers().size() ); + assertEquals( "check known consumers", 9, repoScanning.getKnownContentConsumers().size() ); assertEquals( "check invalid consumers", 1, repoScanning.getInvalidContentConsumers().size() ); List patterns = filetypes.getFileTypePatterns( "artifacts" ); diff --git a/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java b/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java index e5e91b27d..9ec52b8f7 100644 --- a/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java +++ b/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java @@ -75,7 +75,7 @@ public class ArtifactMissingChecksumsConsumer extends AbstractMonitoredConsumer /** * @plexus.requirement role="org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayout" */ - private Map bidirectionalLayoutMap; + private Map bidirectionalLayoutMap; // TODO: replace with new bidir-repo-layout-factory /** * @plexus.requirement role-hint="sha1" @@ -137,7 +137,7 @@ public void beginScan( ArchivaRepository repository ) throws ConsumerException if ( !bidirectionalLayoutMap.containsKey( layoutName ) ) { throw new ConsumerException( "Unable to process repository with layout [" + layoutName - + "] as there is no coresponding " + BidirectionalRepositoryLayout.class.getName() + + "] as there is no corresponding " + BidirectionalRepositoryLayout.class.getName() + " implementation available." ); } diff --git a/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java b/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java new file mode 100644 index 000000000..10410cf4e --- /dev/null +++ b/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/maven/archiva/consumers/core/MetadataUpdaterConsumer.java @@ -0,0 +1,287 @@ +package org.apache.maven.archiva.consumers.core; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.archiva.configuration.ArchivaConfiguration; +import org.apache.maven.archiva.configuration.ConfigurationNames; +import org.apache.maven.archiva.configuration.FileTypes; +import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer; +import org.apache.maven.archiva.consumers.ConsumerException; +import org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer; +import org.apache.maven.archiva.model.ArchivaRepository; +import org.apache.maven.archiva.model.ArtifactReference; +import org.apache.maven.archiva.model.ProjectReference; +import org.apache.maven.archiva.model.VersionedReference; +import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayout; +import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayoutFactory; +import org.apache.maven.archiva.repository.layout.LayoutException; +import org.apache.maven.archiva.repository.metadata.MetadataTools; +import org.apache.maven.archiva.repository.metadata.RepositoryMetadataException; +import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable; +import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException; +import org.codehaus.plexus.registry.Registry; +import org.codehaus.plexus.registry.RegistryListener; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * MetadataUpdaterConsumer will create and update the metadata present within the repository. + * + * @author Joakim Erdfelt + * @version $Id$ + + * @plexus.component role="org.apache.maven.archiva.consumers.KnownRepositoryContentConsumer" + * role-hint="metadata-updater" + * instantiation-strategy="per-lookup" + */ +public class MetadataUpdaterConsumer + extends AbstractMonitoredConsumer + implements KnownRepositoryContentConsumer, RegistryListener, Initializable +{ + /** + * @plexus.configuration default-value="metadata-updater" + */ + private String id; + + /** + * @plexus.configuration default-value="Update / Create maven-metadata.xml files" + */ + private String description; + + /** + * @plexus.requirement + */ + private BidirectionalRepositoryLayoutFactory layoutFactory; + + /** + * @plexus.requirement + */ + private MetadataTools metadataTools; + + /** + * @plexus.requirement + */ + private ArchivaConfiguration configuration; + + /** + * @plexus.requirement + */ + private FileTypes filetypes; + + private static final String TYPE_METADATA_BAD_INTERNAL_REF = "metadata-bad-internal-ref"; + + private static final String TYPE_METADATA_WRITE_FAILURE = "metadata-write-failure"; + + private static final String TYPE_METADATA_IO = "metadata-io-warning"; + + private ArchivaRepository repository; + + private File repositoryDir; + + private BidirectionalRepositoryLayout repositoryLayout; + + private List includes = new ArrayList(); + + private long scanStartTimestamp = 0; + + public String getDescription() + { + return description; + } + + public String getId() + { + return id; + } + + public void setIncludes( List includes ) + { + this.includes = includes; + } + + public void beginScan( ArchivaRepository repository ) + throws ConsumerException + { + if ( !repository.isManaged() ) + { + throw new ConsumerException( "Consumer requires managed repository." ); + } + + this.repository = repository; + this.repositoryDir = new File( repository.getUrl().getPath() ); + try + { + this.repositoryLayout = layoutFactory.getLayout( repository.getLayoutType() ); + } + catch ( LayoutException e ) + { + throw new ConsumerException( "Cannot operate with bad layout definition on repo [" + repository.getId() + + "]: " + e.getMessage(), e ); + } + this.scanStartTimestamp = System.currentTimeMillis(); + } + + public void completeScan() + { + /* do nothing here */ + } + + public List getExcludes() + { + return null; + } + + public List getIncludes() + { + return this.includes; + } + + public void processFile( String path ) + throws ConsumerException + { + try + { + ArtifactReference artifact = this.repositoryLayout.toArtifactReference( path ); + updateVersionMetadata( artifact, path ); + updateProjectMetadata( artifact, path ); + } + catch ( LayoutException e ) + { + throw new ConsumerException( "Unable to convert to artifact reference: " + path, e ); + } + } + + private void updateProjectMetadata( ArtifactReference artifact, String path ) + { + ProjectReference projectRef = new ProjectReference(); + projectRef.setGroupId( artifact.getGroupId() ); + projectRef.setArtifactId( artifact.getArtifactId() ); + + try + { + String metadataPath = this.metadataTools.toPath( projectRef ); + + File projectMetadata = new File( this.repositoryDir, metadataPath ); + + if ( projectMetadata.exists() && ( projectMetadata.lastModified() >= this.scanStartTimestamp ) ) + { + // This metadata is up to date. skip it. + getLogger().debug( "Skipping uptodate metadata: " + this.metadataTools.toPath( projectRef ) ); + return; + } + + metadataTools.updateMetadata( this.repository, projectRef ); + getLogger().info( "Updated metadata: " + this.metadataTools.toPath( projectRef ) ); + } + catch ( LayoutException e ) + { + triggerConsumerWarning( TYPE_METADATA_BAD_INTERNAL_REF, "Unable to convert path [" + path + + "] to an internal project reference: " + e.getMessage() ); + } + catch ( RepositoryMetadataException e ) + { + triggerConsumerError( TYPE_METADATA_WRITE_FAILURE, "Unable to write project metadata for artifact [" + path + + "]: " + e.getMessage() ); + } + catch ( IOException e ) + { + triggerConsumerWarning( TYPE_METADATA_IO, "Project metadata not written due to IO warning: " + + e.getMessage() ); + } + } + + private void updateVersionMetadata( ArtifactReference artifact, String path ) + { + VersionedReference versionRef = new VersionedReference(); + versionRef.setGroupId( artifact.getGroupId() ); + versionRef.setArtifactId( artifact.getArtifactId() ); + versionRef.setVersion( artifact.getVersion() ); + + try + { + String metadataPath = this.metadataTools.toPath( versionRef ); + + File projectMetadata = new File( this.repositoryDir, metadataPath ); + + if ( projectMetadata.exists() && ( projectMetadata.lastModified() >= this.scanStartTimestamp ) ) + { + // This metadata is up to date. skip it. + getLogger().debug( "Skipping uptodate metadata: " + this.metadataTools.toPath( versionRef ) ); + return; + } + + metadataTools.updateMetadata( this.repository, versionRef ); + getLogger().info( "Updated metadata: " + this.metadataTools.toPath( versionRef ) ); + } + catch ( LayoutException e ) + { + triggerConsumerWarning( TYPE_METADATA_BAD_INTERNAL_REF, "Unable to convert path [" + path + + "] to an internal version reference: " + e.getMessage() ); + } + catch ( RepositoryMetadataException e ) + { + triggerConsumerError( TYPE_METADATA_WRITE_FAILURE, "Unable to write version metadata for artifact [" + path + + "]: " + e.getMessage() ); + } + catch ( IOException e ) + { + triggerConsumerWarning( TYPE_METADATA_IO, "Version metadata not written due to IO warning: " + + e.getMessage() ); + } + } + + public boolean isPermanent() + { + return false; + } + + public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue ) + { + getLogger().info( "After Configuration Change: propertyName [" + propertyName + "]" ); + + if ( ConfigurationNames.isRepositoryScanning( propertyName ) ) + { + initIncludes(); + } + } + + public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue ) + { + /* do nothing here */ + } + + private void initIncludes() + { + includes.clear(); + + includes.addAll( filetypes.getFileTypePatterns( FileTypes.ARTIFACTS ) ); + } + + public void initialize() + throws InitializationException + { + configuration.addChangeListener( this ); + + initIncludes(); + } +} diff --git a/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java b/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java index 9808be7cc..005d229e8 100644 --- a/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java +++ b/archiva-base/archiva-proxy/src/main/java/org/apache/maven/archiva/proxy/DefaultRepositoryProxyConnectors.java @@ -174,7 +174,7 @@ public File fetchFromProxies( ArchivaRepository repository, VersionedReference m { ProxyConnector connector = (ProxyConnector) it.next(); ArchivaRepository targetRepository = connector.getTargetRepository(); - String targetPath = getLayout( targetRepository ).toPath( metadata ); + String targetPath = metadataTools.toPath( metadata ); File localRepoFile = toLocalRepoFile( repository, targetRepository, targetPath ); File downloadedFile = transferFile( connector, targetRepository, targetPath, localRepoFile, requestProperties ); @@ -240,7 +240,7 @@ public File fetchFromProxies( ArchivaRepository repository, ProjectReference met { ProxyConnector connector = (ProxyConnector) it.next(); ArchivaRepository targetRepository = connector.getTargetRepository(); - String targetPath = getLayout( targetRepository ).toPath( metadata ); + String targetPath = metadataTools.toPath( metadata ); File localRepoFile = toLocalRepoFile( repository, targetRepository, targetPath ); File downloadedFile = transferFile( connector, targetRepository, targetPath, localRepoFile, requestProperties ); @@ -328,16 +328,14 @@ private File toLocalFile( ArchivaRepository repository, ArtifactReference artifa private File toLocalFile( ArchivaRepository repository, ProjectReference metadata ) throws ProxyException { - BidirectionalRepositoryLayout sourceLayout = getLayout( repository ); - String sourcePath = sourceLayout.toPath( metadata ); + String sourcePath = metadataTools.toPath( metadata ); return new File( repository.getUrl().getPath(), sourcePath ); } private File toLocalFile( ArchivaRepository repository, VersionedReference metadata ) throws ProxyException { - BidirectionalRepositoryLayout sourceLayout = getLayout( repository ); - String sourcePath = sourceLayout.toPath( metadata ); + String sourcePath = metadataTools.toPath( metadata ); return new File( repository.getUrl().getPath(), sourcePath ); } diff --git a/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/AbstractProxyTestCase.java b/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/AbstractProxyTestCase.java index 73077e926..7c498a1dd 100644 --- a/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/AbstractProxyTestCase.java +++ b/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/AbstractProxyTestCase.java @@ -27,8 +27,6 @@ import org.apache.maven.archiva.model.ArchivaArtifact; import org.apache.maven.archiva.model.ArchivaRepository; import org.apache.maven.archiva.model.ArtifactReference; -import org.apache.maven.archiva.model.ProjectReference; -import org.apache.maven.archiva.model.VersionedReference; import org.apache.maven.archiva.policies.urlcache.UrlFailureCache; import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayout; import org.apache.maven.archiva.repository.layout.BidirectionalRepositoryLayoutFactory; @@ -256,22 +254,6 @@ protected ArchivaRepository createManagedLegacyRepository() "Test Managed (Legacy) Repository", "legacy" ); } - protected ProjectReference createProjectReference( String layoutType, String path ) - throws Exception - { - BidirectionalRepositoryLayout layout = layoutFactory.getLayout( layoutType ); - ProjectReference metadata = layout.toProjectReference( path ); - return metadata; - } - - protected VersionedReference createVersionedReference( String layoutType, String path ) - throws Exception - { - BidirectionalRepositoryLayout layout = layoutFactory.getLayout( layoutType ); - VersionedReference metadata = layout.toVersionedReference( path ); - return metadata; - } - protected ArchivaRepository createProxiedLegacyRepository() { return createRepository( "src/test/repositories/legacy-proxied", "testProxiedLegacyRepo", diff --git a/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/MetadataTransferTest.java b/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/MetadataTransferTest.java index 52e9605e2..e562b307c 100644 --- a/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/MetadataTransferTest.java +++ b/archiva-base/archiva-proxy/src/test/java/org/apache/maven/archiva/proxy/MetadataTransferTest.java @@ -677,7 +677,7 @@ private void assertFetchProject( String requestedResource ) { File expectedFile = new File( managedDefaultDir, requestedResource ); - ProjectReference metadata = createProjectReference( "default", requestedResource ); + ProjectReference metadata = createProjectReference( requestedResource ); File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata ); @@ -685,6 +685,12 @@ private void assertFetchProject( String requestedResource ) assertNoTempFiles( expectedFile ); } + private ProjectReference createProjectReference( String path ) + throws RepositoryMetadataException + { + return metadataTools.toProjectReference( path ); + } + /** * Transfer the metadata file, not expected to succeed. * @@ -695,7 +701,7 @@ private void assertFetchProjectFailed( String requestedResource ) throws Exception { File expectedFile = new File( managedDefaultDir, requestedResource ); - ProjectReference metadata = createProjectReference( "default", requestedResource ); + ProjectReference metadata = createProjectReference( requestedResource ); File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata ); @@ -714,7 +720,7 @@ private void assertFetchVersioned( String requestedResource ) { File expectedFile = new File( managedDefaultDir, requestedResource ); - VersionedReference metadata = createVersionedReference( "default", requestedResource ); + VersionedReference metadata = createVersionedReference( requestedResource ); File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata ); @@ -722,6 +728,12 @@ private void assertFetchVersioned( String requestedResource ) assertNoTempFiles( expectedFile ); } + private VersionedReference createVersionedReference( String path ) + throws RepositoryMetadataException + { + return metadataTools.toVersionedReference( path ); + } + /** * Transfer the metadata file, not expected to succeed. * @@ -732,7 +744,7 @@ private void assertFetchVersionedFailed( String requestedResource ) throws Exception { File expectedFile = new File( managedDefaultDir, requestedResource ); - VersionedReference metadata = createVersionedReference( "default", requestedResource ); + VersionedReference metadata = createVersionedReference( requestedResource ); File downloadedFile = proxyHandler.fetchFromProxies( managedDefaultRepository, metadata ); @@ -813,7 +825,7 @@ private void assertProjectMetadataContents( String requestedResource, String exp File actualFile = new File( managedDefaultDir, requestedResource ); assertTrue( actualFile.exists() ); - ProjectReference metadata = createProjectReference( "default", requestedResource ); + ProjectReference metadata = createProjectReference( requestedResource ); // Build expected metadata XML StringWriter expectedMetadataXml = new StringWriter(); @@ -846,7 +858,7 @@ private void assertReleaseMetadataContents( String requestedResource ) File actualFile = new File( managedDefaultDir, requestedResource ); assertTrue( "Release Metadata should exist: " + requestedResource, actualFile.exists() ); - VersionedReference metadata = createVersionedReference( "default", requestedResource ); + VersionedReference metadata = createVersionedReference( requestedResource ); // Build expected metadata XML StringWriter expectedMetadataXml = new StringWriter(); @@ -879,7 +891,7 @@ private void assertSnapshotMetadataContents( String requestedResource, String ex File actualFile = new File( managedDefaultDir, requestedResource ); assertTrue( "Snapshot Metadata should exist: " + requestedResource, actualFile.exists() ); - VersionedReference actualMetadata = createVersionedReference( "default", requestedResource ); + VersionedReference actualMetadata = createVersionedReference( requestedResource ); assertSnapshotMetadata( actualFile, actualMetadata, expectedDate, expectedTime, expectedBuildnumber ); } @@ -905,7 +917,7 @@ private void assertRepoSnapshotMetadataContents( String proxiedRepoId, String re File actualFile = new File( managedDefaultDir, proxiedFile ); assertTrue( "Repo Specific Snapshot Metadata should exist: " + requestedResource, actualFile.exists() ); - VersionedReference actualMetadata = createVersionedReference( "default", requestedResource ); + VersionedReference actualMetadata = createVersionedReference( requestedResource ); assertSnapshotMetadata( actualFile, actualMetadata, expectedDate, expectedTime, expectedBuildnumber ); } @@ -955,7 +967,7 @@ private void assertRepoProjectMetadata( String proxiedRepoId, String requestedRe File actualFile = new File( managedDefaultDir, proxiedFile ); assertTrue( actualFile.exists() ); - ProjectReference metadata = createProjectReference( "default", requestedResource ); + ProjectReference metadata = createProjectReference( requestedResource ); // Build expected metadata XML StringWriter expectedMetadataXml = new StringWriter(); @@ -990,7 +1002,7 @@ private void assertRepoReleaseMetadataContents( String proxiedRepoId, String req File actualFile = new File( managedDefaultDir, proxiedFile ); assertTrue( "Release metadata for repo should exist: " + actualFile, actualFile.exists() ); - VersionedReference metadata = createVersionedReference( "default", requestedResource ); + VersionedReference metadata = createVersionedReference( requestedResource ); // Build expected metadata XML StringWriter expectedMetadataXml = new StringWriter();