Apache
/
archiva
mirror of https://github.com/apache/archiva.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switching to new repository API

This commit is contained in:
Martin Stockhammer 2020-06-16 21:24:16 +02:00
parent add9451d9e
commit 1201ef3974
7 changed files with 342 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/pom.xml
View File

@ -65,10 +65,6 @@
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-storage-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-storage-fs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-model</artifactId>

40
archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/MetadataUpdaterConsumer.java
View File

@ -19,23 +19,23 @@ package org.apache.archiva.consumers.core;
* under the License.
*/
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.configuration.FileTypes;
import org.apache.archiva.consumers.AbstractMonitoredConsumer;
import org.apache.archiva.consumers.ConsumerException;
import org.apache.archiva.consumers.KnownRepositoryContentConsumer;
import org.apache.archiva.model.ProjectReference;
import org.apache.archiva.model.VersionedReference;
import org.apache.archiva.repository.ManagedRepositoryContent;
import org.apache.archiva.repository.BaseRepositoryContentLayout;
import org.apache.archiva.repository.LayoutException;
import org.apache.archiva.repository.ManagedRepository;
import org.apache.archiva.repository.BaseRepositoryContentLayout;
import org.apache.archiva.repository.ManagedRepositoryContent;
import org.apache.archiva.repository.RepositoryException;
import org.apache.archiva.repository.RepositoryNotFoundException;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.archiva.repository.content.Artifact;
import org.apache.archiva.repository.metadata.base.MetadataTools;
import org.apache.archiva.repository.content.Project;
import org.apache.archiva.repository.metadata.RepositoryMetadataException;
import org.apache.archiva.repository.metadata.base.MetadataTools;
import org.apache.archiva.repository.storage.StorageAsset;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -74,9 +74,6 @@ public class MetadataUpdaterConsumer
@Inject
private MetadataTools metadataTools;
@Inject
private ArchivaConfiguration configuration;
@Inject
private FileTypes filetypes;
@ -195,24 +192,22 @@ public class MetadataUpdaterConsumer
private void updateProjectMetadata( Artifact artifact, String path )
{
ProjectReference projectRef = new ProjectReference( );
projectRef.setGroupId( artifact.getNamespace( ).getId() );
projectRef.setArtifactId( artifact.getId( ) );
try
{
String metadataPath = this.metadataTools.toPath( projectRef );
Project proj = artifact.getProject( );
String metadataPath = repository.toPath( proj );
StorageAsset projectMetadata = this.repositoryDir.resolve( metadataPath );
if ( projectMetadata.exists() && ( projectMetadata.getModificationTime().toEpochMilli() >= this.scanStartTimestamp ) )
{
// This metadata is up to date. skip it.
log.debug( "Skipping uptodate metadata: {}", this.metadataTools.toPath( projectRef ) );
log.debug( "Skipping uptodate metadata: {}", metadataPath );
return;
}
metadataTools.updateMetadata( this.repository, metadataPath );
log.debug( "Updated metadata: {}", this.metadataTools.toPath( projectRef ) );
log.debug( "Updated metadata: {}", metadataPath );
}
catch ( RepositoryMetadataException e )
{
@ -253,23 +248,6 @@ public class MetadataUpdaterConsumer
}
}
/*
@Override
public void afterConfigurationChange( Registry registry, String propertyName, Object propertyValue )
{
if ( ConfigurationNames.isRepositoryScanning( propertyName ) )
{
initIncludes();
}
}
@Override
public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue )
{
// do nothing here
}
*/
private void initIncludes( )
{
includes = new ArrayList<>( filetypes.getFileTypePatterns( FileTypes.ARTIFACTS ) );

19
archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/CleanupReleasedSnapshotsRepositoryPurge.java
View File

@ -204,18 +204,19 @@ public class CleanupReleasedSnapshotsRepositoryPurge
@SuppressWarnings( "deprecation" )
private void updateMetadata( Artifact artifact )
{
VersionedReference versionRef = new VersionedReference( );
versionRef.setGroupId( artifact.getNamespace().getId( ) );
versionRef.setArtifactId( artifact.getId( ) );
versionRef.setVersion( artifact.getVersion().getId( ) );
ProjectReference projectRef = new ProjectReference( );
projectRef.setGroupId( artifact.getNamespace().getId( ) );
projectRef.setArtifactId( artifact.getId( ) );
ItemSelector versionRef = ArchivaItemSelector.builder( )
.withNamespace( artifact.getNamespace( ).getId( ) )
.withProjectId( artifact.getId( ) )
.withVersion( artifact.getVersion( ).getId( ) ).build( );
ItemSelector projectRef = ArchivaItemSelector.builder()
.withNamespace( artifact.getNamespace().getId( ) )
.withProjectId( artifact.getId( ) ).build();
try
{
metadataTools.updateMetadata( repository, versionRef );
metadataTools.updateVersionMetadata( repository, versionRef );
}
catch ( ContentNotFoundException e )
{
@ -228,7 +229,7 @@ public class CleanupReleasedSnapshotsRepositoryPurge
try
{
metadataTools.updateMetadata( repository, projectRef );
metadataTools.updateProjectMetadata( repository, projectRef );
}
catch ( ContentNotFoundException | RepositoryMetadataException | IOException | LayoutException e )
{

244
archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/metadata/base/MetadataTools.java
View File

@ -41,9 +41,11 @@ import org.apache.archiva.repository.ContentNotFoundException;
import org.apache.archiva.repository.LayoutException;
import org.apache.archiva.repository.ManagedRepositoryContent;
import org.apache.archiva.repository.RemoteRepositoryContent;
import org.apache.archiva.repository.RepositoryContent;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.archiva.repository.RepositoryType;
import org.apache.archiva.repository.content.Artifact;
import org.apache.archiva.repository.content.ContentItem;
import org.apache.archiva.repository.content.ItemSelector;
import org.apache.archiva.repository.content.Project;
import org.apache.archiva.repository.content.base.ArchivaItemSelector;
@ -150,6 +152,81 @@ public class MetadataTools
/* nothing to do */
}
/**
* Gather the set of snapshot versions found in a particular versioned reference.
*
* @return the Set of snapshot artifact versions found.
* @throws LayoutException
* @throws ContentNotFoundException
*/
public Set<String> gatherSnapshotVersions( ManagedRepositoryContent managedRepository,
ItemSelector reference )
throws LayoutException, IOException, ContentNotFoundException
{
Set<String> foundVersions = null;
try
{
ArchivaItemSelector selector = ArchivaItemSelector.builder( )
.withNamespace( reference.getNamespace() )
.withProjectId( reference.getArtifactId( ) )
.withArtifactId( reference.getArtifactId( ) )
.withVersion( reference.getVersion( ) )
.build( );
try(Stream<? extends Artifact> stream = managedRepository.getLayout( BaseRepositoryContentLayout.class ).newArtifactStream( selector )) {
foundVersions = stream.map( a -> a.getArtifactVersion( ) )
.filter( StringUtils::isNotEmpty )
.collect( Collectors.toSet( ) );
}
}
catch ( org.apache.archiva.repository.ContentAccessException e )
{
log.error( "Error while accessing content {}", e.getMessage( ) );
throw new IOException( "Could not access repository content: " + e.getMessage( ) );
}
// Next gather up the referenced 'latest' versions found in any proxied repositories
// maven-metadata-${proxyId}.xml files that may be present.
// Does this repository have a set of remote proxied repositories?
Set<String> proxiedRepoIds = this.proxies.get( managedRepository.getId() );
if ( CollectionUtils.isNotEmpty( proxiedRepoIds ) )
{
String baseVersion = VersionUtil.getBaseVersion( reference.getVersion() );
baseVersion = baseVersion.substring( 0, baseVersion.indexOf( VersionUtil.SNAPSHOT ) - 1 );
// Add in the proxied repo version ids too.
Iterator<String> it = proxiedRepoIds.iterator();
while ( it.hasNext() )
{
String proxyId = it.next();
ArchivaRepositoryMetadata proxyMetadata = readProxyMetadata( managedRepository, reference, proxyId );
if ( proxyMetadata == null )
{
// There is no proxy metadata, skip it.
continue;
}
// Is there some snapshot info?
SnapshotVersion snapshot = proxyMetadata.getSnapshotVersion();
if ( snapshot != null )
{
String timestamp = snapshot.getTimestamp();
int buildNumber = snapshot.getBuildNumber();
// Only interested in the timestamp + buildnumber.
if ( StringUtils.isNotBlank( timestamp ) && ( buildNumber > 0 ) )
{
foundVersions.add( baseVersion + "-" + timestamp + "-" + buildNumber );
}
}
}
}
return foundVersions;
}
/**
* Gather the set of snapshot versions found in a particular versioned reference.
*
@ -314,6 +391,25 @@ public class MetadataTools
return reference;
}
public String toPath( ContentItem reference )
{
return reference.getAsset().resolve( MAVEN_METADATA ).getPath();
}
public String toPath( ItemSelector reference )
{
StringBuilder path = new StringBuilder();
path.append( formatAsDirectory( reference.getNamespace() ) ).append( PATH_SEPARATOR );
path.append( reference.getProjectId() ).append( PATH_SEPARATOR );
if (reference.hasVersion()) {
path.append( reference.getVersion( ) ).append( PATH_SEPARATOR );
}
path.append( MAVEN_METADATA );
return path.toString( );
}
public String toPath( ProjectReference reference )
@ -395,6 +491,15 @@ public class MetadataTools
configuration.addListener( this );
}
public ArchivaRepositoryMetadata readProxyMetadata( ManagedRepositoryContent managedRepository,
ItemSelector reference, String proxyId )
{
String metadataPath = getRepositorySpecificName( proxyId, toPath( reference ) );
StorageAsset metadataFile = managedRepository.getRepository().getAsset( metadataPath );
return readMetadataFile( managedRepository, metadataFile );
}
public ArchivaRepositoryMetadata readProxyMetadata( ManagedRepositoryContent managedRepository,
ProjectReference reference, String proxyId )
{
@ -537,7 +642,7 @@ public class MetadataTools
* @throws ContentNotFoundException
* @deprecated
*/
public void updateMetadata( ManagedRepositoryContent managedRepository, ProjectReference reference )
public void updateProjectMetadata( ManagedRepositoryContent managedRepository, ItemSelector reference )
throws LayoutException, RepositoryMetadataException, IOException, ContentNotFoundException
{
@ -548,13 +653,13 @@ public class MetadataTools
long lastUpdated = getExistingLastUpdated( existingMetadata );
ArchivaRepositoryMetadata metadata = new ArchivaRepositoryMetadata();
metadata.setGroupId( reference.getGroupId() );
metadata.setArtifactId( reference.getArtifactId() );
metadata.setGroupId( reference.getNamespace() );
metadata.setArtifactId( reference.getProjectId() );
// Gather up all versions found in the managed repository.
ItemSelector selector = ArchivaItemSelector.builder( )
.withNamespace( reference.getGroupId( ) )
.withProjectId( reference.getArtifactId( ) )
.withNamespace( reference.getNamespace() )
.withProjectId( reference.getProjectId( ) )
.build();
Set<String> allVersions = null;
try
@ -761,6 +866,135 @@ public class MetadataTools
return getLastUpdated( metadata );
}
/**
* Update the metadata based on the following rules.
* <p>
* 1) If this is a SNAPSHOT reference, then utilize the proxy/repository specific
* metadata files to represent the current / latest SNAPSHOT available.
* 2) If this is a RELEASE reference, and the metadata file does not exist, then
* create the metadata file with contents required of the VersionedReference
*
* @param managedRepository the managed repository where the metadata is kept.
* @param reference the versioned reference to update
* @throws LayoutException
* @throws RepositoryMetadataException
* @throws IOException
* @throws ContentNotFoundException
* @deprecated
*/
public void updateVersionMetadata( ManagedRepositoryContent managedRepository, ItemSelector reference )
throws LayoutException, RepositoryMetadataException, IOException, ContentNotFoundException
{
StorageAsset metadataFile = managedRepository.getRepository().getAsset( toPath( reference ) );
ArchivaRepositoryMetadata existingMetadata = readMetadataFile(managedRepository, metadataFile );
long lastUpdated = getExistingLastUpdated( existingMetadata );
ArchivaRepositoryMetadata metadata = new ArchivaRepositoryMetadata();
metadata.setGroupId( reference.getNamespace() );
metadata.setArtifactId( reference.getArtifactId() );
if ( VersionUtil.isSnapshot( reference.getVersion() ) )
{
// Do SNAPSHOT handling.
metadata.setVersion( VersionUtil.getBaseVersion( reference.getVersion() ) );
// Gather up all of the versions found in the reference dir, and any
// proxied maven-metadata.xml files.
Set<String> snapshotVersions = gatherSnapshotVersions( managedRepository, reference );
if ( snapshotVersions.isEmpty() )
{
throw new ContentNotFoundException(
"No snapshot versions found on reference [" + reference + "]." );
}
// sort the list to determine to aide in determining the Latest version.
List<String> sortedVersions = new ArrayList<>();
sortedVersions.addAll( snapshotVersions );
Collections.sort( sortedVersions, new VersionComparator() );
String latestVersion = sortedVersions.get( sortedVersions.size() - 1 );
if ( VersionUtil.isUniqueSnapshot( latestVersion ) )
{
// The latestVersion will contain the full version string "1.0-alpha-5-20070821.213044-8"
// This needs to be broken down into ${base}-${timestamp}-${build_number}
Matcher m = VersionUtil.UNIQUE_SNAPSHOT_PATTERN.matcher( latestVersion );
if ( m.matches() )
{
metadata.setSnapshotVersion( new SnapshotVersion() );
int buildNumber = NumberUtils.toInt( m.group( 3 ), -1 );
metadata.getSnapshotVersion().setBuildNumber( buildNumber );
Matcher mtimestamp = VersionUtil.TIMESTAMP_PATTERN.matcher( m.group( 2 ) );
if ( mtimestamp.matches() )
{
String tsDate = mtimestamp.group( 1 );
String tsTime = mtimestamp.group( 2 );
long snapshotLastUpdated = toLastUpdatedLong( tsDate + tsTime );
lastUpdated = Math.max( lastUpdated, snapshotLastUpdated );
metadata.getSnapshotVersion().setTimestamp( m.group( 2 ) );
}
}
}
else if ( VersionUtil.isGenericSnapshot( latestVersion ) )
{
// The latestVersion ends with the generic version string.
// Example: 1.0-alpha-5-SNAPSHOT
metadata.setSnapshotVersion( new SnapshotVersion() );
/* Disabled due to decision in [MRM-535].
* Do not set metadata.lastUpdated to file.lastModified.
*
* Should this be the last updated timestamp of the file, or in the case of an
* archive, the most recent timestamp in the archive?
*
ArtifactReference artifact = getFirstArtifact( managedRepository, reference );
if ( artifact == null )
{
throw new IOException( "Not snapshot artifact found to reference in " + reference );
}
File artifactFile = managedRepository.toFile( artifact );
if ( artifactFile.exists() )
{
Date lastModified = new Date( artifactFile.lastModified() );
metadata.setLastUpdatedTimestamp( lastModified );
}
*/
}
else
{
throw new RepositoryMetadataException(
"Unable to process snapshot version <" + latestVersion + "> reference <" + reference + ">" );
}
}
else
{
// Do RELEASE handling.
metadata.setVersion( reference.getVersion() );
}
// Set last updated
if ( lastUpdated > 0 )
{
metadata.setLastUpdatedTimestamp( toLastUpdatedDate( lastUpdated ) );
}
// Save the metadata model to disk.
RepositoryMetadataWriter.write( metadata, metadataFile );
ChecksummedFile checksum = new ChecksummedFile( metadataFile.getFilePath() );
checksum.fixChecksums( algorithms );
}
/**
* Update the metadata based on the following rules.
* <p>

4
archiva-modules/archiva-base/archiva-repository-scanner/pom.xml
View File

@ -65,10 +65,6 @@
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-filelock</artifactId>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-model</artifactId>
</dependency>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-repository-layer</artifactId>

4
archiva-modules/archiva-base/archiva-xml-tools/pom.xml
View File

@ -41,10 +41,6 @@
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-storage-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>

97
archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/metadata/MetadataToolsTest.java
View File

@ -21,6 +21,8 @@ package org.apache.archiva.repository.maven.metadata;
import org.apache.archiva.common.utils.VersionComparator;
import org.apache.archiva.configuration.ProxyConnectorConfiguration;
import org.apache.archiva.repository.ManagedRepositoryContent;
import org.apache.archiva.repository.content.ItemSelector;
import org.apache.archiva.repository.content.base.ArchivaItemSelector;
import org.apache.archiva.repository.maven.metadata.storage.mock.MockConfiguration;
import org.apache.archiva.model.ProjectReference;
import org.apache.archiva.model.VersionedReference;
@ -162,17 +164,17 @@ public class MetadataToolsTest
throws Exception
{
ManagedRepositoryContent testRepo = createTestRepoContent();
ProjectReference reference = new ProjectReference();
reference.setGroupId( "org.apache.archiva.metadata.tests" );
reference.setArtifactId( "missing_artifact" );
ItemSelector reference = ArchivaItemSelector.builder()
.withNamespace( "org.apache.archiva.metadata.tests" )
.withProjectId( "missing_artifact" ).build();
prepTestRepo( testRepo, reference );
prepProjectTestRepo( testRepo, reference );
// check metadata prior to update -- should contain the non-existing artifact version
assertProjectMetadata( testRepo, reference, "missing_artifact",
new String[]{ "1.0-SNAPSHOT", "1.1-SNAPSHOT", "1.2-SNAPSHOT" }, "1.2-SNAPSHOT", null );
tools.updateMetadata( testRepo, reference );
tools.updateProjectMetadata( testRepo, reference );
// metadata should not contain the non-existing artifact version -- 1.1-SNAPSHOT
assertProjectMetadata( testRepo, reference, "missing_artifact", new String[]{ "1.0-SNAPSHOT", "1.2-SNAPSHOT" },
@ -403,6 +405,24 @@ public class MetadataToolsTest
}
}
private void assertProjectMetadata( String expectedMetadata, ManagedRepositoryContent repository,
ItemSelector reference )
throws LayoutException, IOException, SAXException, ParserConfigurationException
{
Path metadataFile = repository.getRepository().getRoot().getFilePath().resolve(tools.toPath( reference ) );
String actualMetadata = org.apache.archiva.common.utils.FileUtils.readFileToString( metadataFile, Charset.defaultCharset() );
Diff detailedDiff = DiffBuilder.compare( expectedMetadata ).withTest( actualMetadata ).checkForSimilar().build();
if ( detailedDiff.hasDifferences() )
{
for ( Difference diff : detailedDiff.getDifferences() ) {
System.out.println( diff );
}
// If it isn't similar, dump the difference.
assertEquals( expectedMetadata, actualMetadata );
}
}
private void assertMetadata( String expectedMetadata, ManagedRepositoryContent repository,
ProjectReference reference )
throws LayoutException, IOException, SAXException, ParserConfigurationException
@ -449,18 +469,18 @@ public class MetadataToolsTest
throws Exception
{
ManagedRepositoryContent testRepo = createTestRepoContent();
ProjectReference reference = new ProjectReference();
reference.setGroupId( "org.apache.archiva.metadata.tests" );
reference.setArtifactId( artifactId );
ItemSelector reference = ArchivaItemSelector.builder( )
.withNamespace( "org.apache.archiva.metadata.tests" )
.withProjectId( artifactId ).build();
prepTestRepo( testRepo, reference );
prepProjectTestRepo( testRepo, reference );
tools.updateMetadata( testRepo, reference );
tools.updateProjectMetadata( testRepo, reference );
StringBuilder buf = new StringBuilder();
buf.append( "<metadata>\n" );
buf.append( " <groupId>" ).append( reference.getGroupId() ).append( "</groupId>\n" );
buf.append( " <artifactId>" ).append( reference.getArtifactId() ).append( "</artifactId>\n" );
buf.append( " <groupId>" ).append( reference.getNamespace() ).append( "</groupId>\n" );
buf.append( " <artifactId>" ).append( reference.getProjectId() ).append( "</artifactId>\n" );
// buf.append( " <version>1.0</version>\n" );
if ( expectedVersions != null )
@ -485,7 +505,42 @@ public class MetadataToolsTest
}
buf.append( "</metadata>" );
assertMetadata( buf.toString(), testRepo, reference );
assertProjectMetadata( buf.toString(), testRepo, reference );
}
private void assertProjectMetadata( ManagedRepositoryContent testRepo, ItemSelector reference,
String artifactId, String[] expectedVersions, String latestVersion,
String releaseVersion )
throws Exception
{
StringBuilder buf = new StringBuilder();
buf.append( "<metadata>\n" );
buf.append( " <groupId>" ).append( reference.getNamespace() ).append( "</groupId>\n" );
buf.append( " <artifactId>" ).append( reference.getProjectId() ).append( "</artifactId>\n" );
if ( expectedVersions != null )
{
buf.append( " <versioning>\n" );
if ( latestVersion != null )
{
buf.append( " <latest>" ).append( latestVersion ).append( "</latest>\n" );
}
if ( releaseVersion != null )
{
buf.append( " <release>" ).append( releaseVersion ).append( "</release>\n" );
}
buf.append( " <versions>\n" );
for ( int i = 0; i < expectedVersions.length; i++ )
{
buf.append( " <version>" ).append( expectedVersions[i] ).append( "</version>\n" );
}
buf.append( " </versions>\n" );
buf.append( " </versioning>\n" );
}
buf.append( "</metadata>" );
assertProjectMetadata( buf.toString(), testRepo, reference );
}
private void assertProjectMetadata( ManagedRepositoryContent testRepo, ProjectReference reference,
@ -648,6 +703,22 @@ public class MetadataToolsTest
return repoContent;
}
private void prepProjectTestRepo( ManagedRepositoryContent repo, ItemSelector reference)
throws IOException
{
String groupDir = StringUtils.replaceChars( reference.getNamespace(), '.', '/' );
String path = groupDir + "/" + reference.getArtifactId();
Path srcRepoDir = getRepositoryPath( "metadata-repository" );
Path srcDir = srcRepoDir.resolve( path );
Path destDir = repo.getRepository().getRoot().getFilePath().resolve( path );
assertTrue( "Source Dir exists: " + srcDir, Files.exists(srcDir) );
Files.createDirectories(destDir);
FileUtils.copyDirectory( srcDir.toFile(), destDir.toFile() );
}
private void prepTestRepo( ManagedRepositoryContent repo, ProjectReference reference )
throws IOException
{