[MRM-125] add timestamp checking and tests for metadata discovery

git-svn-id: https://svn.apache.org/repos/asf/maven/repository-manager/trunk@425025 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Brett Porter 2006-07-24 12:39:02 +00:00
parent bf826a5063
commit 7927d9a184
15 changed files with 341 additions and 122 deletions

View File

@ -26,6 +26,7 @@ import org.apache.maven.repository.discovery.DiscovererException;
import org.apache.maven.repository.discovery.MetadataDiscoverer;
import org.apache.maven.repository.indexing.ArtifactRepositoryIndex;
import org.apache.maven.repository.indexing.MetadataRepositoryIndex;
import org.apache.maven.repository.indexing.RepositoryIndex;
import org.apache.maven.repository.indexing.RepositoryIndexException;
import org.apache.maven.repository.indexing.RepositoryIndexingFactory;
import org.codehaus.plexus.logging.AbstractLogEnabled;
@ -152,7 +153,7 @@ public class IndexerTask
try
{
ArtifactRepository repository = repoFactory.createRepository( configuration );
ArtifactRepositoryIndex artifactIndex = indexFactory.createArtifactRepositoryIndex( indexPath, repository );
RepositoryIndex artifactIndex = indexFactory.createArtifactRepositoryIndex( indexPath, repository );
if ( !artifactIndex.indexExists() )
{
execute( configuration, indexPath );

View File

@ -18,19 +18,13 @@ package org.apache.maven.repository.discovery;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.repository.ArtifactRepository;
import org.codehaus.plexus.util.IOUtil;
import org.codehaus.plexus.util.xml.Xpp3Dom;
import org.codehaus.plexus.util.xml.Xpp3DomWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
/**
* Base class for artifact discoverers.
@ -49,8 +43,6 @@ public abstract class AbstractArtifactDiscoverer
"**/*.MD5", "**/*.sha1", "**/*.SHA1", "**/*snapshot-version", "*/website/**", "*/licenses/**", "*/licences/**",
"**/.htaccess", "**/*.html", "**/*.asc", "**/*.txt", "**/*.xml", "**/README*", "**/CHANGELOG*", "**/KEYS*"};
private static final String POM = ".pom";
private List scanForArtifactPaths( File repositoryBase, String blacklistedPatterns, long comparisonTimestamp )
{
return scanForArtifactPaths( repositoryBase, blacklistedPatterns, null, STANDARD_DISCOVERY_EXCLUDES,
@ -113,78 +105,6 @@ public abstract class AbstractArtifactDiscoverer
return artifacts;
}
public void resetLastCheckedTime( ArtifactRepository repository, String operation )
throws IOException
{
// TODO: get these changes into maven-metadata.xml and migrate towards that. The model is further diverging to a different layout at each level so submodels might be a good idea.
// TODO: maven-artifact probably needs an improved pathOfMetadata to cope with top level metadata
// TODO: might we need to write this as maven-metadata-local in some circumstances? merge others? Probably best to keep it simple and just use this format at the root. No need to merge anything that I can see
// TODO: since this metadata isn't meant to be shared, perhaps another file is called for after all.
// Format is: <repository><lastDiscovery><KEY>yyyyMMddHHmmss</KEY></lastDiscovery></repository> (ie, flat properties)
File file = new File( repository.getBasedir(), "maven-metadata.xml" );
Xpp3Dom dom = readDom( file );
Xpp3Dom lastDiscoveryDom = getLastDiscoveryDom( dom );
boolean changed = false;
// do this in reverse so that removing doesn't affect counter
Xpp3Dom[] children = lastDiscoveryDom.getChildren();
for ( int i = lastDiscoveryDom.getChildCount() - 1; i >= 0; i-- )
{
if ( children[i].getName().equals( operation ) )
{
changed = true;
lastDiscoveryDom.removeChild( i );
}
}
if ( changed )
{
saveDom( file, dom );
}
}
private void saveDom( File file, Xpp3Dom dom )
throws IOException
{
FileWriter writer = new FileWriter( file );
// save metadata
try
{
Xpp3DomWriter.write( writer, dom );
}
finally
{
IOUtil.close( writer );
}
}
public void setLastCheckedTime( ArtifactRepository repository, String operation, Date date )
throws IOException
{
// see notes in resetLastCheckedTime
File file = new File( repository.getBasedir(), "maven-metadata.xml" );
Xpp3Dom dom = readDom( file );
Xpp3Dom lastDiscoveryDom = getLastDiscoveryDom( dom );
Xpp3Dom entry = lastDiscoveryDom.getChild( operation );
if ( entry == null )
{
entry = new Xpp3Dom( operation );
lastDiscoveryDom.addChild( entry );
}
entry.setValue( new SimpleDateFormat( DATE_FMT, Locale.US ).format( date ) );
saveDom( file, dom );
}
/**
* Returns an artifact object that is represented by the specified path in a repository
*

View File

@ -25,16 +25,19 @@ import org.codehaus.plexus.util.IOUtil;
import org.codehaus.plexus.util.StringUtils;
import org.codehaus.plexus.util.xml.Xpp3Dom;
import org.codehaus.plexus.util.xml.Xpp3DomBuilder;
import org.codehaus.plexus.util.xml.Xpp3DomWriter;
import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@ -201,4 +204,76 @@ public abstract class AbstractDiscoverer
}
return lastDiscoveryDom;
}
public void resetLastCheckedTime( ArtifactRepository repository, String operation )
throws IOException
{
// TODO: get these changes into maven-metadata.xml and migrate towards that. The model is further diverging to a different layout at each level so submodels might be a good idea.
// TODO: maven-artifact probably needs an improved pathOfMetadata to cope with top level metadata
// TODO: might we need to write this as maven-metadata-local in some circumstances? merge others? Probably best to keep it simple and just use this format at the root. No need to merge anything that I can see
// TODO: since this metadata isn't meant to be shared, perhaps another file is called for after all.
// Format is: <repository><lastDiscovery><KEY>yyyyMMddHHmmss</KEY></lastDiscovery></repository> (ie, flat properties)
File file = new File( repository.getBasedir(), "maven-metadata.xml" );
Xpp3Dom dom = readDom( file );
Xpp3Dom lastDiscoveryDom = getLastDiscoveryDom( dom );
boolean changed = false;
// do this in reverse so that removing doesn't affect counter
Xpp3Dom[] children = lastDiscoveryDom.getChildren();
for ( int i = lastDiscoveryDom.getChildCount() - 1; i >= 0; i-- )
{
if ( children[i].getName().equals( operation ) )
{
changed = true;
lastDiscoveryDom.removeChild( i );
}
}
if ( changed )
{
saveDom( file, dom );
}
}
private void saveDom( File file, Xpp3Dom dom )
throws IOException
{
FileWriter writer = new FileWriter( file );
// save metadata
try
{
Xpp3DomWriter.write( writer, dom );
}
finally
{
IOUtil.close( writer );
}
}
public void setLastCheckedTime( ArtifactRepository repository, String operation, Date date )
throws IOException
{
// see notes in resetLastCheckedTime
File file = new File( repository.getBasedir(), "maven-metadata.xml" );
Xpp3Dom dom = readDom( file );
Xpp3Dom lastDiscoveryDom = getLastDiscoveryDom( dom );
Xpp3Dom entry = lastDiscoveryDom.getChild( operation );
if ( entry == null )
{
entry = new Xpp3Dom( operation );
lastDiscoveryDom.addChild( entry );
}
entry.setValue( new SimpleDateFormat( DATE_FMT, Locale.US ).format( date ) );
saveDom( file, dom );
}
}

View File

@ -19,8 +19,6 @@ package org.apache.maven.repository.discovery;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.repository.ArtifactRepository;
import java.io.IOException;
import java.util.Date;
import java.util.List;
/**
@ -63,25 +61,4 @@ public interface ArtifactDiscoverer
*/
Artifact buildArtifact( String path )
throws DiscovererException;
/**
* Reset the time in the repository that indicates the last time a check was performed.
*
* @param repository the location of the repository
* @param operation the operation to record the timestamp for
* @throws java.io.IOException if there is a non-recoverable problem reading or writing the metadata
*/
void resetLastCheckedTime( ArtifactRepository repository, String operation )
throws IOException;
/**
* Set the time in the repository that indicates the last time a check was performed.
*
* @param repository the location of the repository
* @param operation the operation to record the timestamp for
* @param date the date to set the last check to
* @throws java.io.IOException if there is a non-recoverable problem reading or writing the metadata
*/
void setLastCheckedTime( ArtifactRepository repository, String operation, Date date )
throws IOException;
}

View File

@ -36,6 +36,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.StringTokenizer;
@ -51,21 +52,39 @@ public class DefaultMetadataDiscoverer
{
/**
* Standard patterns to include in discovery of metadata files.
*
* @todo do we really need all these paths? Add tests for all 3 levels and confirm only 2 are needed.
*/
private static final String[] STANDARD_DISCOVERY_INCLUDES = {"**/*-metadata.xml", "**/*/*-metadata.xml",
"**/*/*/*-metadata.xml", "**/*-metadata-*.xml", "**/*/*-metadata-*.xml", "**/*/*/*-metadata-*.xml"};
private static final String[] STANDARD_DISCOVERY_INCLUDES = {"**/maven-metadata.xml", "**/maven-metadata-*.xml"};
public List discoverMetadata( ArtifactRepository repository, String operation, String blacklistedPatterns )
throws DiscovererException
{
if ( !"file".equals( repository.getProtocol() ) )
{
throw new UnsupportedOperationException( "Only filesystem repositories are supported" );
}
long comparisonTimestamp = readComparisonTimestamp( repository, operation );
// Note that last checked time is deliberately set to the start of the process so that anything added
// mid-discovery and missed by the scanner will get checked next time.
// Due to this, there must be no negative side-effects of discovering something twice.
Date newLastCheckedTime = new Date();
List metadataFiles = new ArrayList();
List metadataPaths = scanForArtifactPaths( new File( repository.getBasedir() ), blacklistedPatterns,
STANDARD_DISCOVERY_INCLUDES, null, comparisonTimestamp );
// TODO: save! should we be using a different entry for metadata?
// Also note that the last check time, while set at the start, is saved at the end, so that if any exceptions
// occur, then the timestamp is not updated so that the discovery is attempted again
// TODO: under the list-return behaviour we have now, exceptions might occur later and the timestamp will not be reset - see MRM-83
try
{
setLastCheckedTime( repository, operation, newLastCheckedTime );
}
catch ( IOException e )
{
throw new DiscovererException( "Error writing metadata: " + e.getMessage(), e );
}
for ( Iterator i = metadataPaths.iterator(); i.hasNext(); )
{
@ -155,7 +174,7 @@ public class DefaultMetadataDiscoverer
Artifact artifact = null;
if ( !StringUtils.isEmpty( metaVersion ) )
{
artifact = artifactFactory.createBuildArtifact( metaGroupId, metaArtifactId, metaVersion, "jar" );
artifact = artifactFactory.createProjectArtifact( metaGroupId, metaArtifactId, metaVersion );
}
// snapshotMetadata
@ -174,6 +193,11 @@ public class DefaultMetadataDiscoverer
{
metadata = new ArtifactRepositoryMetadata( artifact );
}
else
{
artifact = artifactFactory.createProjectArtifact( metaGroupId, metaArtifactId, "1.0" );
metadata = new ArtifactRepositoryMetadata( artifact );
}
}
else
{

View File

@ -1,5 +1,9 @@
package org.apache.maven.repository.discovery;
import org.apache.maven.artifact.repository.ArtifactRepository;
import java.io.IOException;
import java.util.Date;
import java.util.Iterator;
/*
@ -36,4 +40,25 @@ public interface Discoverer
* @return the paths as Strings.
*/
Iterator getExcludedPathsIterator();
/**
* Reset the time in the repository that indicates the last time a check was performed.
*
* @param repository the location of the repository
* @param operation the operation to record the timestamp for
* @throws java.io.IOException if there is a non-recoverable problem reading or writing the metadata
*/
void resetLastCheckedTime( ArtifactRepository repository, String operation )
throws IOException;
/**
* Set the time in the repository that indicates the last time a check was performed.
*
* @param repository the location of the repository
* @param operation the operation to record the timestamp for
* @param date the date to set the last check to
* @throws java.io.IOException if there is a non-recoverable problem reading or writing the metadata
*/
void setLastCheckedTime( ArtifactRepository repository, String operation, Date date )
throws IOException;
}

View File

@ -36,5 +36,6 @@ public interface MetadataDiscoverer
* @param blacklistedPatterns Patterns that are to be excluded from the discovery process.
* @return the list of artifacts found
*/
List discoverMetadata( ArtifactRepository repository, String operation, String blacklistedPatterns );
List discoverMetadata( ArtifactRepository repository, String operation, String blacklistedPatterns )
throws DiscovererException;
}

View File

@ -111,7 +111,7 @@ public abstract class AbstractArtifactDiscovererTest
assertNotNull( "Check artifacts not null", artifacts );
assertFalse( "Check not included",
artifacts.contains( createArtifact( "org.apache.maven.update", "test-not-updated", "1.0" ) ) );
artifacts.contains( createArtifact( "org.apache.maven.update", "test-updated", "1.0" ) ) );
}
public void testNotUpdatedInRepository()

View File

@ -16,14 +16,26 @@ package org.apache.maven.repository.discovery;
* limitations under the License.
*/
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.factory.ArtifactFactory;
import org.apache.maven.artifact.repository.ArtifactRepository;
import org.apache.maven.artifact.repository.ArtifactRepositoryFactory;
import org.apache.maven.artifact.repository.layout.ArtifactRepositoryLayout;
import org.apache.maven.artifact.repository.metadata.ArtifactRepositoryMetadata;
import org.apache.maven.artifact.repository.metadata.GroupRepositoryMetadata;
import org.apache.maven.artifact.repository.metadata.RepositoryMetadata;
import org.apache.maven.artifact.repository.metadata.SnapshotArtifactRepositoryMetadata;
import org.codehaus.plexus.PlexusTestCase;
import org.codehaus.plexus.component.repository.exception.ComponentLookupException;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
/**
* This class tests the DefaultMetadataDiscoverer class.
@ -37,6 +49,8 @@ public class DefaultMetadataDiscovererTest
private ArtifactRepository repository;
private ArtifactFactory factory;
/**
*
*/
@ -47,6 +61,8 @@ public class DefaultMetadataDiscovererTest
discoverer = (MetadataDiscoverer) lookup( MetadataDiscoverer.ROLE, "default" );
factory = (ArtifactFactory) lookup( ArtifactFactory.ROLE );
repository = getRepository();
removeTimestampMetadata();
@ -74,20 +90,11 @@ public class DefaultMetadataDiscovererTest
discoverer = null;
}
/**
* Test DefaultMetadataDiscoverer when the all metadata paths are valid.
*/
public void testMetadataDiscovererSuccess()
{
List metadataPaths = discoverer.discoverMetadata( repository, TEST_OPERATION, null );
assertNotNull( "Check metadata not null", metadataPaths );
assertEquals( 3, metadataPaths.size() );
}
/**
* Test if metadata file in wrong directory was added to the kickedOutPaths.
*/
public void testKickoutWrongDirectory()
throws DiscovererException
{
discoverer.discoverMetadata( repository, TEST_OPERATION, null );
Iterator iter = discoverer.getKickedOutPathsIterator();
@ -112,6 +119,7 @@ public class DefaultMetadataDiscovererTest
* Test if blank metadata file was added to the kickedOutPaths.
*/
public void testKickoutBlankMetadata()
throws DiscovererException
{
discoverer.discoverMetadata( repository, TEST_OPERATION, null );
Iterator iter = discoverer.getKickedOutPathsIterator();
@ -139,4 +147,163 @@ public class DefaultMetadataDiscovererTest
file.delete();
assertFalse( file.exists() );
}
public void testDiscoverMetadata()
throws DiscovererException
{
List metadataPaths = discoverer.discoverMetadata( repository, TEST_OPERATION, null );
assertNotNull( "Check metadata not null", metadataPaths );
RepositoryMetadata metadata =
new ArtifactRepositoryMetadata( createArtifact( "org.apache.testgroup", "discovery" ) );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
metadata =
new SnapshotArtifactRepositoryMetadata( createArtifact( "org.apache.testgroup", "discovery", "1.0" ) );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
metadata = new GroupRepositoryMetadata( "org.apache.maven" );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
}
public void testUpdatedInRepository()
throws ComponentLookupException, DiscovererException, ParseException, IOException
{
// Set repository time to 1-1-2000, a time in the distant past so definitely updated
discoverer.setLastCheckedTime( repository, "update",
new SimpleDateFormat( "yyyy-MM-dd", Locale.US ).parse( "2000-01-01" ) );
List metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
RepositoryMetadata metadata =
new ArtifactRepositoryMetadata( createArtifact( "org.apache.maven.update", "test-updated" ) );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
// try again with the updated timestamp
metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
assertFalse( "Check not included", containsMetadata( metadataPaths, metadata ) );
}
private boolean containsMetadata( List metadataPaths, RepositoryMetadata metadata )
{
for ( Iterator i = metadataPaths.iterator(); i.hasNext(); )
{
RepositoryMetadata m = (RepositoryMetadata) i.next();
if ( m.getGroupId().equals( metadata.getGroupId() ) )
{
if ( m.getArtifactId() == null && metadata.getArtifactId() == null )
{
return true;
}
else if ( m.getArtifactId() != null && m.getArtifactId().equals( metadata.getArtifactId() ) )
{
return true;
}
}
}
return false;
}
public void testNotUpdatedInRepository()
throws ComponentLookupException, DiscovererException, IOException
{
// Set repository time to now, which is after any artifacts, so definitely not updated
discoverer.setLastCheckedTime( repository, "update", new Date() );
List metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
RepositoryMetadata metadata =
new ArtifactRepositoryMetadata( createArtifact( "org.apache.maven.update", "test-not-updated" ) );
assertFalse( "Check not included", containsMetadata( metadataPaths, metadata ) );
}
public void testNotUpdatedInRepositoryForcedDiscovery()
throws ComponentLookupException, DiscovererException, IOException
{
discoverer.resetLastCheckedTime( repository, "update" );
List metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
RepositoryMetadata metadata =
new ArtifactRepositoryMetadata( createArtifact( "org.apache.maven.update", "test-not-updated" ) );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
// try again with the updated timestamp
metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
assertFalse( "Check not included", containsMetadata( metadataPaths, metadata ) );
}
public void testNotUpdatedInRepositoryForcedDiscoveryMetadataAlreadyExists()
throws ComponentLookupException, DiscovererException, IOException
{
discoverer.setLastCheckedTime( repository, "update", new Date() );
discoverer.resetLastCheckedTime( repository, "update" );
List metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
RepositoryMetadata metadata =
new ArtifactRepositoryMetadata( createArtifact( "org.apache.maven.update", "test-not-updated" ) );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
// try again with the updated timestamp
metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
assertFalse( "Check not included", containsMetadata( metadataPaths, metadata ) );
}
public void testNotUpdatedInRepositoryForcedDiscoveryOtherMetadataAlreadyExists()
throws ComponentLookupException, DiscovererException, IOException
{
discoverer.setLastCheckedTime( repository, "test", new Date() );
discoverer.resetLastCheckedTime( repository, "update" );
List metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
RepositoryMetadata metadata =
new ArtifactRepositoryMetadata( createArtifact( "org.apache.maven.update", "test-not-updated" ) );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
// try again with the updated timestamp
metadataPaths = discoverer.discoverMetadata( repository, "update", null );
assertNotNull( "Check metadata not null", metadataPaths );
assertFalse( "Check not included", containsMetadata( metadataPaths, metadata ) );
}
public void testNoRepositoryMetadata()
throws ComponentLookupException, DiscovererException, ParseException, IOException
{
removeTimestampMetadata();
// should find all
List metadataPaths = discoverer.discoverMetadata( repository, TEST_OPERATION, null );
assertNotNull( "Check metadata not null", metadataPaths );
RepositoryMetadata metadata =
new ArtifactRepositoryMetadata( createArtifact( "org.apache.maven.update", "test-updated" ) );
assertTrue( "Check included", containsMetadata( metadataPaths, metadata ) );
}
protected Artifact createArtifact( String groupId, String artifactId )
{
return createArtifact( groupId, artifactId, "1.0" );
}
private Artifact createArtifact( String groupId, String artifactId, String version )
{
return factory.createArtifact( groupId, artifactId, version, null, "jar" );
}
}

View File

@ -0,0 +1,3 @@
<metadata>
<groupId>org.apache.maven</groupId>
</metadata>

View File

@ -0,0 +1,4 @@
<metadata>
<groupId>org.apache.maven.update</groupId>
<artifactId>test-not-updated</artifactId>
</metadata>

View File

@ -0,0 +1,4 @@
<metadata>
<groupId>org.apache.maven.update</groupId>
<artifactId>test-updated</artifactId>
</metadata>

View File

@ -0,0 +1,5 @@
<metadata>
<groupId>org.apache.testgroup</groupId>
<artifactId>discovery</artifactId>
<version>1.0</version>
</metadata>

View File

@ -0,0 +1,4 @@
<metadata>
<groupId>org.apache.testgroup</groupId>
<artifactId>discovery</artifactId>
</metadata>

View File

@ -140,4 +140,13 @@ public interface RepositoryIndex
*/
void deleteDocuments( List termList )
throws RepositoryIndexException, IOException;
/**
* Check if the index already exists.
*
* @return true if the index already exists
* @throws RepositoryIndexException
*/
boolean indexExists()
throws RepositoryIndexException;
}