[MRM-730]

-used bcel to read the .class files
-index classnames (packagename already included here) and methods that are public
-added tests


git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@674740 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Maria Odea B. Ching 2008-07-08 08:44:21 +00:00
parent c5b51473ba
commit 8b9ec9fa88
8 changed files with 620 additions and 18 deletions

View File

@ -49,6 +49,11 @@
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-spring</artifactId>
<scope>test</scope>
</dependency>
</dependency>
<dependency>
<groupId>org.apache.bcel</groupId>
<artifactId>bcel</artifactId>
<version>5.2</version>
</dependency>
</dependencies>
</project>

View File

@ -22,14 +22,34 @@ package org.apache.maven.archiva.consumers.lucene;
import org.apache.maven.archiva.consumers.AbstractMonitoredConsumer;
import org.apache.maven.archiva.consumers.ConsumerException;
import org.apache.maven.archiva.consumers.DatabaseUnprocessedArtifactConsumer;
import org.apache.maven.archiva.indexer.RepositoryContentIndex;
import org.apache.maven.archiva.indexer.RepositoryContentIndexFactory;
import org.apache.maven.archiva.indexer.RepositoryIndexException;
import org.apache.maven.archiva.indexer.bytecode.BytecodeRecord;
import org.apache.maven.archiva.model.ArchivaArtifact;
import org.apache.maven.archiva.repository.ManagedRepositoryContent;
import org.apache.maven.archiva.repository.RepositoryContentFactory;
import org.apache.maven.archiva.repository.RepositoryException;
import com.sun.org.apache.bcel.internal.classfile.ClassParser;
import com.sun.org.apache.bcel.internal.classfile.JavaClass;
import com.sun.org.apache.bcel.internal.classfile.Method;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
/**
* IndexJavaPublicMethodsConsumer
*
* @author <a href="mailto:joakime@apache.org">Joakim Erdfelt</a>
* <a href="mailto:oching@apache.org">Maria Odea Ching</a>
* @version $Id$
*
* @plexus.component role="org.apache.maven.archiva.consumers.DatabaseUnprocessedArtifactConsumer"
@ -49,11 +69,40 @@ public class IndexJavaPublicMethodsConsumer
* @plexus.configuration default-value="Index the java public methods for Full Text Search."
*/
private String description;
/**
* @plexus.requirement role-hint="lucene"
*/
private RepositoryContentIndexFactory repoIndexFactory;
/**
* @plexus.requirement
*/
private RepositoryContentFactory repoFactory;
private static final String CLASSES = "classes";
private static final String METHODS = "methods";
private List<String> includes = new ArrayList<String>();
public IndexJavaPublicMethodsConsumer()
{
includes.add( "jar" );
includes.add( "war" );
includes.add( "ear" );
includes.add( "zip" );
includes.add( "tar.gz" );
includes.add( "tar.bz2" );
includes.add( "car" );
includes.add( "sar" );
includes.add( "mar" );
includes.add( "rar" );
}
public void beginScan()
{
// TODO Auto-generated method stub
// TODO Auto-generated method stubx
}
public void completeScan()
@ -63,16 +112,58 @@ public class IndexJavaPublicMethodsConsumer
}
public List<String> getIncludedTypes()
{
// TODO Auto-generated method stub
return null;
{
return includes;
}
public void processArchivaArtifact( ArchivaArtifact artifact )
throws ConsumerException
{
// TODO Auto-generated method stub
{
try
{
ManagedRepositoryContent repoContent =
repoFactory.getManagedRepositoryContent( artifact.getModel().getRepositoryId() );
File file = new File( repoContent.getRepoRoot(), repoContent.toPath( artifact ) );
if( file.getAbsolutePath().endsWith( ".jar" ) || file.getAbsolutePath().endsWith( ".war" ) ||
file.getAbsolutePath().endsWith( ".ear" ) || file.getAbsolutePath().endsWith( ".zip" ) ||
file.getAbsolutePath().endsWith( ".tar.gz" ) || file.getAbsolutePath().endsWith( ".tar.bz2" ) ||
file.getAbsolutePath().endsWith( ".car" ) || file.getAbsolutePath().endsWith( ".sar" ) ||
file.getAbsolutePath().endsWith( ".mar" ) || file.getAbsolutePath().endsWith( ".rar" ) )
{
if( file.exists() )
{
List<String> files = readFilesInArchive( file );
Map<String, List<String>> mapOfClassesAndMethods =
getPublicClassesAndMethodsFromFiles( file.getAbsolutePath(), files );
// NOTE: what about public variables? should these be indexed too?
RepositoryContentIndex bytecodeIndex = repoIndexFactory.createBytecodeIndex( repoContent.getRepository() );
BytecodeRecord bytecodeRecord = new BytecodeRecord();
bytecodeRecord.setFilename( file.getName() );
bytecodeRecord.setClasses( mapOfClassesAndMethods.get( CLASSES ) );
bytecodeRecord.setFiles( files );
bytecodeRecord.setMethods( mapOfClassesAndMethods.get( METHODS ) );
bytecodeRecord.setArtifact( artifact );
bytecodeRecord.setRepositoryId( artifact.getModel().getRepositoryId() );
bytecodeIndex.modifyRecord( bytecodeRecord );
}
}
}
catch ( RepositoryException e )
{
throw new ConsumerException( "Can't run index cleanup consumer: " + e.getMessage() );
}
catch ( RepositoryIndexException e )
{
throw new ConsumerException( "Error encountered while adding artifact to index: " + e.getMessage() );
}
catch ( IOException e )
{
throw new ConsumerException( "Error encountered while getting file contents: " + e.getMessage() );
}
}
public String getDescription()
@ -89,5 +180,89 @@ public class IndexJavaPublicMethodsConsumer
{
return false;
}
private List<String> readFilesInArchive( File file )
throws IOException
{
ZipFile zipFile = new ZipFile( file );
List<String> files;
try
{
files = new ArrayList<String>( zipFile.size() );
for ( Enumeration entries = zipFile.entries(); entries.hasMoreElements(); )
{
ZipEntry entry = (ZipEntry) entries.nextElement();
files.add( entry.getName() );
}
}
finally
{
closeQuietly( zipFile );
}
return files;
}
private void closeQuietly( ZipFile zipFile )
{
try
{
if ( zipFile != null )
{
zipFile.close();
}
}
catch ( IOException e )
{
// ignored
}
}
private static boolean isClass( String name )
{
return name.endsWith( ".class" ) && name.lastIndexOf( "$" ) < 0;
}
private Map<String, List<String>> getPublicClassesAndMethodsFromFiles( String zipFile, List<String> files )
{
Map<String, List<String>> map = new HashMap<String, List<String>>();
List<String> methods = new ArrayList<String>();
List<String> classes = new ArrayList<String>();
for( String file : files )
{
if( isClass( file ) )
{
try
{
ClassParser parser = new ClassParser( zipFile, file );
JavaClass javaClass = parser.parse();
if( javaClass.isPublic() )
{
classes.add( javaClass.getClassName() );
}
Method[] methodsArr = javaClass.getMethods();
for( Method method : methodsArr )
{
if( method.isPublic() )
{
methods.add( method.getName() );
}
}
}
catch ( IOException e )
{
// ignore
}
}
}
map.put( CLASSES, classes );
map.put( METHODS, methods );
return map;
}
}

View File

@ -0,0 +1,124 @@
package org.apache.maven.archiva.consumers.lucene;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.util.ArrayList;
import java.util.List;
import org.apache.maven.archiva.configuration.ArchivaConfiguration;
import org.apache.maven.archiva.configuration.Configuration;
import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
import org.apache.maven.archiva.consumers.DatabaseUnprocessedArtifactConsumer;
import org.apache.maven.archiva.indexer.RepositoryContentIndexFactory;
import org.apache.maven.archiva.indexer.search.SearchResultLimits;
import org.apache.maven.archiva.indexer.search.SearchResults;
import org.apache.maven.archiva.model.ArchivaArtifact;
import org.apache.maven.archiva.model.ArchivaArtifactModel;
import org.codehaus.plexus.spring.PlexusInSpringTestCase;
/**
*
* @author <a href="mailto:oching@apache.org">Maria Odea Ching</a>
* @version
*
*/
public class IndexJavaPublicMethodsConsumerTest
extends PlexusInSpringTestCase
{
DatabaseUnprocessedArtifactConsumer indexMethodsConsumer;
IndexJavaPublicMethodsCrossRepositorySearch searcher;
private RepositoryContentIndexFactory indexFactory;
public void setUp()
throws Exception
{
super.setUp();
indexMethodsConsumer =
(DatabaseUnprocessedArtifactConsumer) lookup( DatabaseUnprocessedArtifactConsumer.class,
"index-public-methods" );
ManagedRepositoryConfiguration config = new ManagedRepositoryConfiguration();
config.setId( "test-repo" );
config.setLayout( "default" );
config.setLocation( getBasedir() + "/target/test-classes/test-repo" );
config.setName( "Test Repository" );
addRepoToConfiguration( "index-public-methods", config );
indexFactory = (RepositoryContentIndexFactory) lookup (RepositoryContentIndexFactory.class, "lucene" );
searcher = new IndexJavaPublicMethodsCrossRepositorySearch( config, indexFactory );
}
private void addRepoToConfiguration( String configHint, ManagedRepositoryConfiguration repoConfiguration )
throws Exception
{
ArchivaConfiguration archivaConfiguration =
(ArchivaConfiguration) lookup( ArchivaConfiguration.class, configHint );
Configuration configuration = archivaConfiguration.getConfiguration();
configuration.removeManagedRepository( configuration.findManagedRepositoryById( repoConfiguration.getId() ) );
configuration.addManagedRepository( repoConfiguration );
}
public void testJarPublicMethods()
throws Exception
{
ArchivaArtifact artifact =
createArtifact( "org.apache.archiva", "archiva-index-methods-jar-test", "1.0", "jar" );
indexMethodsConsumer.processArchivaArtifact( artifact );
List<String> selectedRepos = new ArrayList<String>();
selectedRepos.add( "test-repo" );
// search for class names
SearchResults results = searcher.searchForBytecode( "", selectedRepos, "FirstPackageApp", new SearchResultLimits( 0 ) );
assertEquals( 1, results.getTotalHits() );
results = searcher.searchForBytecode( "", selectedRepos, "SecondPackageApp", new SearchResultLimits( 0 ) );
assertEquals( 1, results.getTotalHits() );
// search for public methods
results = searcher.searchForBytecode( "", selectedRepos, "appMethodOne", new SearchResultLimits( 0 ) );
assertEquals( 1, results.getTotalHits() );
// should return only the overridding public method in SecondPackageApp
results = searcher.searchForBytecode( "", selectedRepos, "protectedMethod", new SearchResultLimits( 0 ) );
assertEquals( 1, results.getTotalHits() );
// should not return any private methods
results = searcher.searchForBytecode( "", selectedRepos, "privMethod", new SearchResultLimits( 0 ) );
assertEquals( 0, results.getTotalHits() );
// test for public variables?
}
private ArchivaArtifact createArtifact( String groupId, String artifactId, String version, String type )
{
ArchivaArtifactModel model = new ArchivaArtifactModel();
model.setGroupId( groupId );
model.setArtifactId( artifactId );
model.setVersion( version );
model.setType( type );
model.setRepositoryId( "test-repo" );
return new ArchivaArtifact( model );
}
}

View File

@ -0,0 +1,216 @@
package org.apache.maven.archiva.consumers.lucene;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.MultiSearcher;
import org.apache.lucene.search.Searchable;
import org.apache.maven.archiva.configuration.ManagedRepositoryConfiguration;
import org.apache.maven.archiva.indexer.RepositoryContentIndex;
import org.apache.maven.archiva.indexer.RepositoryContentIndexFactory;
import org.apache.maven.archiva.indexer.RepositoryIndexSearchException;
import org.apache.maven.archiva.indexer.bytecode.BytecodeHandlers;
import org.apache.maven.archiva.indexer.lucene.LuceneEntryConverter;
import org.apache.maven.archiva.indexer.lucene.LuceneQuery;
import org.apache.maven.archiva.indexer.lucene.LuceneRepositoryContentRecord;
import org.apache.maven.archiva.indexer.search.CrossRepositorySearch;
import org.apache.maven.archiva.indexer.search.SearchResultLimits;
import org.apache.maven.archiva.indexer.search.SearchResults;
/**
* Searcher used for testing purposes only.
*
* @author <a href="mailto:oching@apache.org">Maria Odea Ching</a>
* @version
*/
public class IndexJavaPublicMethodsCrossRepositorySearch
implements CrossRepositorySearch
{
private ManagedRepositoryConfiguration localIndexedRepo;
private RepositoryContentIndexFactory indexFactory;
public IndexJavaPublicMethodsCrossRepositorySearch()
{
}
public IndexJavaPublicMethodsCrossRepositorySearch( ManagedRepositoryConfiguration localIndexedRepo, RepositoryContentIndexFactory indexFactory )
{
this.localIndexedRepo = localIndexedRepo;
this.indexFactory = indexFactory;
}
public SearchResults searchForBytecode( String principal, List<String> selectedRepos, String term,
SearchResultLimits limits )
{
List<RepositoryContentIndex> indexes = new ArrayList<RepositoryContentIndex>();
indexes.add( indexFactory.createBytecodeIndex( localIndexedRepo ) );
try
{
QueryParser parser = new BytecodeHandlers().getQueryParser();
LuceneQuery query = new LuceneQuery( parser.parse( term ) );
SearchResults results = searchAll( query, limits, indexes );
results.getRepositories().add( localIndexedRepo );
return results;
}
catch ( ParseException e )
{
}
return new SearchResults();
}
public SearchResults searchForChecksum( String principal, List<String> selectedRepos, String checksum,
SearchResultLimits limits )
{
// TODO Auto-generated method stub
return null;
}
public SearchResults searchForTerm( String principal, List<String> selectedRepos, String term,
SearchResultLimits limits )
{
// TODO Auto-generated method stub
return null;
}
private SearchResults searchAll( LuceneQuery luceneQuery, SearchResultLimits limits, List<RepositoryContentIndex> indexes )
{
org.apache.lucene.search.Query specificQuery = luceneQuery.getLuceneQuery();
SearchResults results = new SearchResults();
if ( indexes.isEmpty() )
{
// No point going any further.
return results;
}
// Setup the converter
LuceneEntryConverter converter = null;
RepositoryContentIndex index = indexes.get( 0 );
converter = index.getEntryConverter();
// Process indexes into an array of Searchables.
List<Searchable> searchableList = toSearchables( indexes );
Searchable searchables[] = new Searchable[searchableList.size()];
searchableList.toArray( searchables );
MultiSearcher searcher = null;
try
{
// Create a multi-searcher for looking up the information.
searcher = new MultiSearcher( searchables );
// Perform the search.
Hits hits = searcher.search( specificQuery );
int hitCount = hits.length();
// Now process the limits.
results.setLimits( limits );
results.setTotalHits( hitCount );
int fetchCount = limits.getPageSize();
int offset = ( limits.getSelectedPage() * limits.getPageSize() );
if ( limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
{
fetchCount = hitCount;
offset = 0;
}
// Goto offset.
if ( offset < hitCount )
{
// only process if the offset is within the hit count.
for ( int i = 0; i <= fetchCount; i++ )
{
// Stop fetching if we are past the total # of available hits.
if ( offset + i >= hitCount )
{
break;
}
try
{
Document doc = hits.doc( offset + i );
LuceneRepositoryContentRecord record = converter.convert( doc );
results.addHit( record );
}
catch ( java.text.ParseException e )
{
}
}
}
}
catch ( IOException e )
{
}
finally
{
try
{
if ( searcher != null )
{
searcher.close();
}
}
catch ( IOException ie )
{
}
}
return results;
}
private List<Searchable> toSearchables( List<RepositoryContentIndex> indexes )
{
List<Searchable> searchableList = new ArrayList<Searchable>();
for ( RepositoryContentIndex contentIndex : indexes )
{
try
{
searchableList.add( contentIndex.getSearchable() );
}
catch ( RepositoryIndexSearchException e )
{
}
}
return searchableList;
}
}

View File

@ -0,0 +1,64 @@
<component-set>
<components>
<!-- for testing repo purge using retention count -->
<component>
<role>org.apache.maven.archiva.consumers.DatabaseUnprocessedArtifactConsumer</role>
<role-hint>index-public-methods</role-hint>
<implementation>org.apache.maven.archiva.consumers.lucene.IndexJavaPublicMethodsConsumer</implementation>
<requirements>
<requirement>
<role>org.apache.maven.archiva.repository.RepositoryContentFactory</role>
<role-hint>index-public-methods</role-hint>
</requirement>
<requirement>
<role>org.apache.maven.archiva.indexer.RepositoryContentIndexFactory</role>
<role-hint>lucene</role-hint>
<field-name>repoIndexFactory</field-name>
</requirement>
</requirements>
</component>
<component>
<role>org.apache.maven.archiva.repository.RepositoryContentFactory</role>
<role-hint>index-public-methods</role-hint>
<implementation>org.apache.maven.archiva.repository.RepositoryContentFactory</implementation>
<description>RepositoryContentRequest</description>
<requirements>
<requirement>
<role>org.apache.maven.archiva.configuration.ArchivaConfiguration</role>
<role-hint>index-public-methods</role-hint>
<field-name>archivaConfiguration</field-name>
</requirement>
</requirements>
</component>
<component>
<role>org.apache.maven.archiva.configuration.ArchivaConfiguration</role>
<role-hint>index-public-methods</role-hint>
<implementation>org.apache.maven.archiva.configuration.DefaultArchivaConfiguration</implementation>
<requirements>
<requirement>
<role>org.codehaus.plexus.registry.Registry</role>
<role-hint>index-public-methods</role-hint>
</requirement>
<requirement>
<role>org.apache.maven.archiva.policies.PreDownloadPolicy</role>
<field-name>prePolicies</field-name>
</requirement>
<requirement>
<role>org.apache.maven.archiva.policies.PostDownloadPolicy</role>
<field-name>postPolicies</field-name>
</requirement>
</requirements>
</component>
<component>
<role>org.codehaus.plexus.registry.Registry</role>
<role-hint>index-public-methods</role-hint>
<implementation>org.codehaus.plexus.registry.commons.CommonsConfigurationRegistry</implementation>
<configuration>
<properties>
<xml fileName="${basedir}/src/test/conf/repository-manager.xml"
config-name="org.apache.maven.archiva" config-at="org.apache.maven.archiva"/>
</properties>
</configuration>
</component>
</components>
</component-set>

View File

@ -0,0 +1,18 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-index-methods-jar-test</artifactId>
<packaging>jar</packaging>
<version>1.0</version>
<name>archiva-index-methods-jar-test</name>
<url>http://maven.apache.org</url>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -39,28 +39,28 @@ public class BytecodeRecord
private String filename;
private List classes;
private List<String> classes;
private List methods;
private List<String> methods;
private List files;
private List<String> files;
public ArchivaArtifact getArtifact()
{
return artifact;
}
public List getClasses()
public List<String> getClasses()
{
return classes;
}
public List getFiles()
public List<String> getFiles()
{
return files;
}
public List getMethods()
public List<String> getMethods()
{
return methods;
}
@ -92,17 +92,17 @@ public class BytecodeRecord
this.artifact = artifact;
}
public void setClasses( List classes )
public void setClasses( List<String> classes )
{
this.classes = classes;
}
public void setFiles( List files )
public void setFiles( List<String> files )
{
this.files = files;
}
public void setMethods( List methods )
public void setMethods( List<String> methods )
{
this.methods = methods;
}