Apache/archiva
1
0
Fork 0
mirror of https://github.com/apache/archiva.git synced 2025-02-21 01:15:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

[MRM-1283] improve performance of on-demand resolution from storage. Still could be improved.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/archiva/branches/MRM-1025@885095 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Brett Porter 2009-11-28 14:28:30 +00:00
parent 61745667ab
commit 3d61eecdc4
10 changed files with 298 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/DefaultMetadataResolver.java
View File

@ -19,12 +19,15 @@
* under the License.
*/
import java.util.ArrayList;
import java.util.Collection;
import org.apache.archiva.metadata.model.Dependency;
import org.apache.archiva.metadata.model.ProjectMetadata;
import org.apache.archiva.metadata.model.ProjectVersionMetadata;
import org.apache.archiva.metadata.model.ProjectVersionReference;
import org.apache.archiva.metadata.repository.filter.ExcludesFilter;
import org.apache.archiva.metadata.repository.storage.StorageMetadataResolver;
/**
* @plexus.component role="org.apache.archiva.metadata.repository.MetadataResolver"
@ -44,7 +47,7 @@ public class DefaultMetadataResolver
*
* @plexus.requirement role-hint="maven2"
*/
private MetadataResolver storageResolver;
private StorageMetadataResolver storageResolver;
public ProjectMetadata getProject( String repoId, String namespace, String projectId )
{
@ -104,56 +107,79 @@ public Collection<ProjectVersionReference> getProjectReferences( String repoId,
public Collection<String> getRootNamespaces( String repoId )
{
Collection<String> rootNamespaces = metadataRepository.getRootNamespaces( repoId );
// TODO: may want caching on this
Collection<String> storageRootNamespaces = storageResolver.getRootNamespaces( repoId );
if ( storageRootNamespaces != null && !storageRootNamespaces.equals( rootNamespaces ) )
Collection<String> namespaces = metadataRepository.getRootNamespaces( repoId );
Collection<String> storageNamespaces =
storageResolver.getRootNamespaces( repoId, new ExcludesFilter<String>( namespaces ) );
if ( storageNamespaces != null && !storageNamespaces.isEmpty() )
{
// TODO: update the metadata repository
rootNamespaces = storageRootNamespaces;
for ( String n : storageNamespaces )
{
metadataRepository.updateNamespace( repoId, n );
}
namespaces = new ArrayList<String>( namespaces );
namespaces.addAll( storageNamespaces );
}
return rootNamespaces;
return namespaces;
}
public Collection<String> getNamespaces( String repoId, String namespace )
{
Collection<String> namespaces = metadataRepository.getNamespaces( repoId, namespace );
// TODO: may want caching on this
Collection<String> storageNamespaces = storageResolver.getNamespaces( repoId, namespace );
if ( storageNamespaces != null && !storageNamespaces.equals( namespaces ) )
Collection<String> storageNamespaces =
storageResolver.getNamespaces( repoId, namespace, new ExcludesFilter<String>( namespaces ) );
if ( storageNamespaces != null && !storageNamespaces.isEmpty() )
{
// TODO: update the metadata repository
namespaces = storageNamespaces;
for ( String n : storageNamespaces )
{
metadataRepository.updateNamespace( repoId, namespace + "." + n );
}
namespaces = new ArrayList<String>( namespaces );
namespaces.addAll( storageNamespaces );
}
return namespaces;
}
public Collection<String> getProjects( String repoId, String namespace )
{
Collection<String> projects = metadataRepository.getProjects( repoId, namespace );
// TODO: may want caching on this
Collection<String> storageProjects = storageResolver.getProjects( repoId, namespace );
if ( storageProjects != null && !storageProjects.equals( projects ) )
Collection<String> storageProjects =
storageResolver.getProjects( repoId, namespace, new ExcludesFilter<String>( projects ) );
if ( storageProjects != null && !storageProjects.isEmpty() )
{
// TODO: update the metadata repository
projects = storageProjects;
for ( String projectId : storageProjects )
{
ProjectMetadata projectMetadata = storageResolver.getProject( repoId, namespace, projectId );
if ( projectMetadata != null )
{
metadataRepository.updateProject( repoId, projectMetadata );
}
}
projects = new ArrayList<String>( projects );
projects.addAll( storageProjects );
}
return projects;
}
public Collection<String> getProjectVersions( String repoId, String namespace, String projectId )
throws MetadataResolverException
{
Collection<String> projectVersions = metadataRepository.getProjectVersions( repoId, namespace, projectId );
// TODO: may want caching on this
Collection<String> storageProjectVersions = storageResolver.getProjectVersions( repoId, namespace, projectId );
if ( storageProjectVersions != null && !storageProjectVersions.equals( projectVersions ) )
Collection<String> storageProjectVersions = storageResolver.getProjectVersions( repoId, namespace, projectId,
new ExcludesFilter<String>(
projectVersions ) );
if ( storageProjectVersions != null && !storageProjectVersions.isEmpty() )
{
// TODO: update the metadata repository
projectVersions = storageProjectVersions;
for ( String projectVersion : storageProjectVersions )
{
ProjectVersionMetadata versionMetadata =
storageResolver.getProjectVersion( repoId, namespace, projectId, projectVersion );
if ( versionMetadata != null )
{
metadataRepository.updateProjectVersion( repoId, namespace, projectId, versionMetadata );
}
}
projectVersions = new ArrayList<String>( projectVersions );
projectVersions.addAll( storageProjectVersions );
}
return projectVersions;
}

3
archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java
View File

@ -30,6 +30,7 @@ public interface MetadataRepository
/**
* Update metadata for a particular project in the metadata repository, or create it if it does not already exist.
*
* @param repoId the repository the project is in
* @param project the project metadata to create or update
*/
void updateProject( String repoId, ProjectMetadata project );
@ -42,4 +43,6 @@ void updateProjectVersion( String repoId, String namespace, String projectId,
void updateProjectReference( String repoId, String namespace, String projectId, String projectVersion,
ProjectVersionReference reference );
void updateNamespace( String repoId, String namespace );
}

3
archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataResolver.java
View File

@ -54,5 +54,6 @@ Collection<ProjectVersionReference> getProjectReferences( String repoId, String
Collection<String> getProjects( String repoId, String namespace );
Collection<String> getProjectVersions( String repoId, String namespace, String projectId );
Collection<String> getProjectVersions( String repoId, String namespace, String projectId )
throws MetadataResolverException;
}

29
archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/filter/AllFilter.java Normal file
View File

@ -0,0 +1,29 @@
package org.apache.archiva.metadata.repository.filter;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
public class AllFilter<T>
implements Filter<T>
{
public boolean accept( T value )
{
return true;
}
}

38
archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/filter/ExcludesFilter.java Normal file
View File

@ -0,0 +1,38 @@
package org.apache.archiva.metadata.repository.filter;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.util.Collection;
public class ExcludesFilter<T>
implements Filter<T>
{
private Collection<T> excludes;
public ExcludesFilter( Collection<T> excludes )
{
this.excludes = excludes;
}
public boolean accept( T value )
{
return !excludes.contains( value );
}
}

25
archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/filter/Filter.java Normal file
View File

@ -0,0 +1,25 @@
package org.apache.archiva.metadata.repository.filter;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
public interface Filter<T>
{
boolean accept( T value );
}

37
archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/storage/StorageMetadataResolver.java Normal file
View File

@ -0,0 +1,37 @@
package org.apache.archiva.metadata.repository.storage;
import java.util.Collection;
import org.apache.archiva.metadata.repository.MetadataResolver;
import org.apache.archiva.metadata.repository.filter.Filter;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
public interface StorageMetadataResolver
extends MetadataResolver
{
Collection<String> getRootNamespaces( String repoId, Filter<String> filter );
Collection<String> getNamespaces( String repoId, String namespace, Filter<String> filter );
Collection<String> getProjects( String repoId, String namespace, Filter<String> filter );
Collection<String> getProjectVersions( String repoId, String namespace, String projectId, Filter<String> filter );
}

137
archiva-modules/plugins/maven2-repository/src/main/java/org/apache/archiva/metadata/repository/storage/maven2/Maven2RepositoryMetadataResolver.java
View File

@ -30,9 +30,11 @@
import org.apache.archiva.metadata.model.ProjectMetadata;
import org.apache.archiva.metadata.model.ProjectVersionMetadata;
import org.apache.archiva.metadata.model.ProjectVersionReference;
import org.apache.archiva.metadata.repository.MetadataResolver;
import org.apache.archiva.metadata.repository.MetadataResolverException;
import org.apache.archiva.metadata.repository.filter.AllFilter;
import org.apache.archiva.metadata.repository.filter.Filter;
import org.apache.archiva.metadata.repository.storage.RepositoryPathTranslator;
import org.apache.archiva.metadata.repository.storage.StorageMetadataResolver;
import org.apache.maven.archiva.common.utils.VersionUtil;
import org.apache.maven.archiva.configuration.ArchivaConfiguration;
import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
@ -53,10 +55,10 @@
import org.slf4j.LoggerFactory;
/**
* @plexus.component role="org.apache.archiva.metadata.repository.MetadataResolver" role-hint="maven2"
* @plexus.component role="org.apache.archiva.metadata.repository.storage.StorageMetadataResolver" role-hint="maven2"
*/
public class Maven2RepositoryMetadataResolver
implements MetadataResolver
implements StorageMetadataResolver
{
/**
* @plexus.requirement
@ -77,25 +79,12 @@ public class Maven2RepositoryMetadataResolver
private static final String METADATA_FILENAME = "maven-metadata.xml";
private static final FilenameFilter DIRECTORY_FILTER = new FilenameFilter()
{
public boolean accept( File dir, String name )
{
if ( name.startsWith( "." ) )
{
return false;
}
else if ( !new File( dir, name ).isDirectory() )
{
return false;
}
return true;
}
};
private static final Filter<String> ALL = new AllFilter<String>();
public ProjectMetadata getProject( String repoId, String namespace, String projectId )
{
throw new UnsupportedOperationException();
// TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
return null;
}
public ProjectVersionMetadata getProjectVersion( String repoId, String namespace, String projectId,
@ -291,24 +280,30 @@ private org.apache.archiva.metadata.model.CiManagement convertCiManagement( CiMa
return ci;
}
// TODO: evidence that storage and resolver != repository API - split the interface up
public Collection<String> getArtifactVersions( String repoId, String namespace, String projectId,
String projectVersion )
{
// TODO: useful, but not implemented yet, not called from DefaultMetadataResolver
throw new UnsupportedOperationException();
}
public Collection<ProjectVersionReference> getProjectReferences( String repoId, String namespace, String projectId,
String projectVersion )
{
// Can't be determined on a Maven 2 repository
throw new UnsupportedOperationException();
}
public Collection<String> getRootNamespaces( String repoId )
{
return getRootNamespaces( repoId, ALL );
}
public Collection<String> getRootNamespaces( String repoId, Filter<String> filter )
{
File dir = getRepositoryBasedir( repoId );
String[] files = dir.list( DIRECTORY_FILTER );
String[] files = dir.list( new DirectoryFilter( filter ) );
return files != null ? Arrays.asList( files ) : Collections.<String>emptyList();
}
@ -321,17 +316,22 @@ private File getRepositoryBasedir( String repoId )
}
public Collection<String> getNamespaces( String repoId, String namespace )
{
return getNamespaces( repoId, namespace, ALL );
}
public Collection<String> getNamespaces( String repoId, String namespace, Filter<String> filter )
{
File dir = pathTranslator.toFile( getRepositoryBasedir( repoId ), namespace );
// scan all the directories which are potential namespaces. Any directories known to be projects are excluded
Collection<String> namespaces = new ArrayList<String>();
File[] files = dir.listFiles( DIRECTORY_FILTER );
File[] files = dir.listFiles( new DirectoryFilter( filter ) );
if ( files != null )
{
for ( File file : files )
{
if ( !isProject( file ) )
if ( !isProject( file, filter ) )
{
namespaces.add( file.getName() );
}
@ -341,17 +341,22 @@ public Collection<String> getNamespaces( String repoId, String namespace )
}
public Collection<String> getProjects( String repoId, String namespace )
{
return getProjects( repoId, namespace, ALL );
}
public Collection<String> getProjects( String repoId, String namespace, Filter<String> filter )
{
File dir = pathTranslator.toFile( getRepositoryBasedir( repoId ), namespace );
// scan all directories in the namespace, and only include those that are known to be projects
Collection<String> projects = new ArrayList<String>();
File[] files = dir.listFiles( DIRECTORY_FILTER );
File[] files = dir.listFiles( new DirectoryFilter( filter ) );
if ( files != null )
{
for ( File file : files )
{
if ( isProject( file ) )
if ( isProject( file, filter ) )
{
projects.add( file.getName() );
}
@ -361,27 +366,24 @@ public Collection<String> getProjects( String repoId, String namespace )
}
public Collection<String> getProjectVersions( String repoId, String namespace, String projectId )
{
return getProjectVersions( repoId, namespace, projectId, ALL );
}
public Collection<String> getProjectVersions( String repoId, String namespace, String projectId,
Filter<String> filter )
{
File dir = pathTranslator.toFile( getRepositoryBasedir( repoId ), namespace, projectId );
// all directories in a project directory can be considered a version
Collection<String> projectVersions = new ArrayList<String>();
String[] files = dir.list( DIRECTORY_FILTER );
String[] files = dir.list( new DirectoryFilter( filter ) );
return files != null ? Arrays.asList( files ) : Collections.<String>emptyList();
}
private boolean isProject( File dir )
private boolean isProject( File dir, Filter<String> filter )
{
// if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
MavenRepositoryMetadata metadata = readMetadata( dir );
if ( metadata != null && dir.getName().equals( metadata.getArtifactId() ) )
{
return true;
}
// if metadata is missing, scan directories for a valid project version subdirectory, meaning this must be a
// project directory
File[] files = dir.listFiles( DIRECTORY_FILTER );
// scan directories for a valid project version subdirectory, meaning this must be a project directory
File[] files = dir.listFiles( new DirectoryFilter( filter ) );
if ( files != null )
{
for ( File file : files )
@ -392,6 +394,14 @@ private boolean isProject( File dir )
}
}
}
// if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
MavenRepositoryMetadata metadata = readMetadata( dir );
if ( metadata != null && dir.getName().equals( metadata.getArtifactId() ) )
{
return true;
}
return false;
}
@ -400,14 +410,7 @@ private boolean isProjectVersion( File dir )
final String artifactId = dir.getParentFile().getName();
final String projectVersion = dir.getName();
// if a metadata file is present, check if this is the "version" directory, marking it as a project version
MavenRepositoryMetadata metadata = readMetadata( dir );
if ( metadata != null && projectVersion.equals( metadata.getVersion() ) )
{
return true;
}
// if metadata is missing, check if there is a POM artifact file to ensure it is a version directory
// check if there is a POM artifact file to ensure it is a version directory
File[] files;
if ( VersionUtil.isSnapshot( projectVersion ) )
{
@ -439,7 +442,19 @@ public boolean accept( File dir, String name )
}
} );
}
return files != null && files.length > 0;
if ( files != null && files.length > 0 )
{
return true;
}
// if a metadata file is present, check if this is the "version" directory, marking it as a project version
MavenRepositoryMetadata metadata = readMetadata( dir );
if ( metadata != null && projectVersion.equals( metadata.getVersion() ) )
{
return true;
}
return false;
}
private MavenRepositoryMetadata readMetadata( File directory )
@ -459,4 +474,32 @@ private MavenRepositoryMetadata readMetadata( File directory )
}
return metadata;
}
private static class DirectoryFilter
implements FilenameFilter
{
private final Filter<String> filter;
public DirectoryFilter( Filter<String> filter )
{
this.filter = filter;
}
public boolean accept( File dir, String name )
{
if ( !filter.accept( name ) )
{
return false;
}
else if ( name.startsWith( "." ) )
{
return false;
}
else if ( !new File( dir, name ).isDirectory() )
{
return false;
}
return true;
}
}
}

4
archiva-modules/plugins/maven2-repository/src/test/java/org/apache/archiva/metadata/repository/storage/maven2/Maven2RepositoryMetadataResolverTest.java
View File

@ -28,8 +28,8 @@
import org.apache.archiva.metadata.model.License;
import org.apache.archiva.metadata.model.MailingList;
import org.apache.archiva.metadata.model.ProjectVersionMetadata;
import org.apache.archiva.metadata.repository.MetadataResolver;
import org.apache.archiva.metadata.repository.MetadataResolverException;
import org.apache.archiva.metadata.repository.storage.StorageMetadataResolver;
import org.apache.maven.archiva.configuration.ArchivaConfiguration;
import org.apache.maven.archiva.configuration.Configuration;
import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
@ -61,7 +61,7 @@ public void setUp()
c.addManagedRepository( testRepo );
configuration.save( c );
resolver = (Maven2RepositoryMetadataResolver) lookup( MetadataResolver.class, "maven2" );
resolver = (Maven2RepositoryMetadataResolver) lookup( StorageMetadataResolver.class, "maven2" );
}
public void testGetProjectVersionMetadata()

21
archiva-modules/plugins/metadata-repository-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java
View File

@ -86,13 +86,13 @@ public void updateProject( String repoId, ProjectMetadata project )
private void updateProject( String repoId, String namespace, String id )
{
// TODO: this is a more braindead implementation than we would normally expect, for prototyping purposes
updateNamespace( repoId, namespace );
try
{
File namespaceDirectory = new File( this.directory, repoId + "/" + namespace );
Properties properties = new Properties();
properties.setProperty( "namespace", namespace );
writeProperties( properties, namespaceDirectory, NAMESPACE_METADATA_KEY );
properties.setProperty( "id", id );
writeProperties( properties, new File( namespaceDirectory, id ), PROJECT_METADATA_KEY );
@ -217,6 +217,23 @@ public void updateProjectReference( String repoId, String namespace, String proj
}
}
public void updateNamespace( String repoId, String namespace )
{
try
{
File namespaceDirectory = new File( this.directory, repoId + "/" + namespace );
Properties properties = new Properties();
properties.setProperty( "namespace", namespace );
writeProperties( properties, namespaceDirectory, NAMESPACE_METADATA_KEY );
}
catch ( IOException e )
{
// TODO!
e.printStackTrace();
}
}
private String join( Collection<String> ids )
{
if ( !ids.isEmpty() )