From 6e252d9a2d88f590189508ca6e8bd8275c74fd57 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Wed, 14 Sep 2011 22:52:52 +0000 Subject: [PATCH] -fix unit : lucene sometimes has pain to flush... git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1170884 13f79535-47bb-0310-9956-ffa450edef68 --- .../archiva-base/archiva-indexer/pom.xml | 15 +------ .../indexer/search/NexusRepositorySearch.java | 18 +++++--- .../indexer/search/SearchResultHit.java | 42 +++++++++++++++++++ .../archiva/indexer/search/SearchResults.java | 23 +++++++--- .../archiva/indexer/util/SearchUtil.java | 9 +++- .../search/AbstractNexusRepositorySearch.java | 39 ++++++++++------- .../search/NexusRepositorySearchTest.java | 39 ++++++++--------- .../indexing/ArchivaIndexingTaskExecutor.java | 2 - 8 files changed, 122 insertions(+), 65 deletions(-) diff --git a/archiva-modules/archiva-base/archiva-indexer/pom.xml b/archiva-modules/archiva-base/archiva-indexer/pom.xml index 2c61d74f6..944d89e83 100644 --- a/archiva-modules/archiva-base/archiva-indexer/pom.xml +++ b/archiva-modules/archiva-base/archiva-indexer/pom.xml @@ -100,19 +100,6 @@ - - - org.codehaus.mojo - cobertura-maven-plugin - - - - 80 - 80 - - - - @@ -120,7 +107,7 @@ apache-rat-plugin - src/test/resources/nexus-search-test-repo*/** + src/test/nexus-search-test-repo*/** diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java index b374b9a9e..86c9230d8 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/NexusRepositorySearch.java @@ -189,8 +189,9 @@ public class NexusRepositorySearch if ( StringUtils.isNotBlank( searchFields.getClassifier() ) ) { - q.add( indexer.constructQuery( MAVEN.CLASSIFIER, new StringSearchExpression( searchFields.getClassifier() ) ), - Occur.MUST ); + q.add( + indexer.constructQuery( MAVEN.CLASSIFIER, new StringSearchExpression( searchFields.getClassifier() ) ), + Occur.MUST ); } if ( q.getClauses() == null || q.getClauses().length <= 0 ) @@ -204,6 +205,7 @@ public class NexusRepositorySearch private SearchResults search( SearchResultLimits limits, BooleanQuery q, List indexingContextIds ) throws RepositorySearchException { + try { FlatSearchRequest request = new FlatSearchRequest( q ); @@ -343,9 +345,11 @@ public class NexusRepositorySearch { log.warn( "IO error occured while accessing index of repository '" + repo + "' : " + e.getMessage() ); continue; - } catch ( RepositoryAdminException e ) + } + catch ( RepositoryAdminException e ) { - log.warn( "RepositoryAdminException occured while accessing index of repository '" + repo + "' : " + e.getMessage() ); + log.warn( "RepositoryAdminException occured while accessing index of repository '" + repo + "' : " + + e.getMessage() ); continue; } } @@ -366,7 +370,8 @@ public class NexusRepositorySearch for ( ArtifactInfo artifactInfo : artifactInfos ) { - String id = SearchUtil.getHitId( artifactInfo.groupId, artifactInfo.artifactId ); + String id = SearchUtil.getHitId( artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier, + artifactInfo.packaging ); Map hitsMap = results.getHitsMap(); SearchResultHit hit = hitsMap.get( id ); @@ -450,7 +455,8 @@ public class NexusRepositorySearch SearchResultHit hit = results.getHits().get( ( offset + i ) ); if ( hit != null ) { - String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId() ); + String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(), + hit.getPackaging() ); paginated.addHit( id, hit ); } else diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResultHit.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResultHit.java index 30119b3c1..0cfbec314 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResultHit.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResultHit.java @@ -373,5 +373,47 @@ public class SearchResultHit return sb.toString(); } + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( o == null || getClass() != o.getClass() ) + { + return false; + } + SearchResultHit that = (SearchResultHit) o; + + if ( artifactId != null ? !artifactId.equals( that.artifactId ) : that.artifactId != null ) + { + return false; + } + if ( classifier != null ? !classifier.equals( that.classifier ) : that.classifier != null ) + { + return false; + } + if ( groupId != null ? !groupId.equals( that.groupId ) : that.groupId != null ) + { + return false; + } + if ( packaging != null ? !packaging.equals( that.packaging ) : that.packaging != null ) + { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = groupId != null ? groupId.hashCode() : 0; + result = 31 * result + ( artifactId != null ? artifactId.hashCode() : 0 ); + result = 31 * result + ( packaging != null ? packaging.hashCode() : 0 ); + result = 31 * result + ( classifier != null ? classifier.hashCode() : 0 ); + return result; + } } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResults.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResults.java index 7ac087239..bd8d65a81 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResults.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/search/SearchResults.java @@ -26,7 +26,7 @@ import java.util.Map; /** - * SearchResults + * SearchResults * * @version $Id: SearchResults.java 742859 2009-02-10 05:35:05Z jdumay $ */ @@ -47,20 +47,31 @@ public class SearchResults // for new RepositorySearch public void addHit( String id, SearchResultHit hit ) - { + { hits.put( id, hit ); } /** * Get the list of {@link SearchResultHit} objects. - * + * * @return the list of {@link SearchResultHit} objects. */ public List getHits() { return new ArrayList( hits.values() ); } - + + /** + * see SearchUtil on how to generate the key + * + * @param key + * @return + */ + public SearchResultHit getSearchResultHit( String key ) + { + return hits.get( key ); + } + public Map getHitsMap() { return hits; @@ -92,8 +103,8 @@ public class SearchResults } /** - * @since 1.4 * @return + * @since 1.4 */ public int getReturnedHitsCount() { @@ -101,8 +112,8 @@ public class SearchResults } /** - * @since 1.4 * @param returnedHitsCount + * @since 1.4 */ public void setReturnedHitsCount( int returnedHitsCount ) { diff --git a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/util/SearchUtil.java b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/util/SearchUtil.java index 2048e6002..bdb4de4ae 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/util/SearchUtil.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/main/java/org/apache/archiva/indexer/util/SearchUtil.java @@ -19,13 +19,18 @@ package org.apache.archiva.indexer.util; * under the License. */ +import org.apache.commons.lang.StringUtils; + /** * SearchUtil - utility class for search. */ public class SearchUtil { - public static String getHitId( String groupId, String artifactId ) + public static String getHitId( String groupId, String artifactId, String classifier, String packaging ) { - return groupId + ":" + artifactId; + return ( StringUtils.isBlank( groupId ) ? "" : StringUtils.trim( groupId ) ) + ":" + + ( StringUtils.isBlank( artifactId ) ? "" : StringUtils.trim( artifactId ) ) + ":" + + ( StringUtils.isBlank( classifier ) ? "" : StringUtils.trim( classifier ) ) + ":" + ( StringUtils.isBlank( + packaging ) ? "" : StringUtils.trim( packaging ) ); } } diff --git a/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/AbstractNexusRepositorySearch.java b/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/AbstractNexusRepositorySearch.java index a10cc5f40..701d39786 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/AbstractNexusRepositorySearch.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/AbstractNexusRepositorySearch.java @@ -46,7 +46,6 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import javax.inject.Inject; import java.io.File; -import java.util.ArrayList; import java.util.List; /** @@ -122,6 +121,12 @@ public abstract class AbstractNexusRepositorySearch nexusIndexer.removeIndexingContext( indexingContext, true ); } + FileUtils.deleteDirectory( new File( FileUtil.getBasedir(), "/target/repos/" + TEST_REPO_1 ) ); + assertFalse( new File( FileUtil.getBasedir(), "/target/repos/" + TEST_REPO_1 ).exists() ); + + FileUtils.deleteDirectory( new File( FileUtil.getBasedir(), "/target/repos/" + TEST_REPO_2 ) ); + assertFalse( new File( FileUtil.getBasedir(), "/target/repos/" + TEST_REPO_2 ).exists() ); + super.tearDown(); } @@ -172,38 +177,40 @@ public abstract class AbstractNexusRepositorySearch assertFalse( lockFile.exists() ); - File repo = new File( FileUtil.getBasedir(), "/target/repos/" + repository ); - File indexDirectory = new File( FileUtil.getBasedir(), "/target/repos/" + repository + "/.indexer" ); + File repo = new File( FileUtil.getBasedir(), "src/test/" + repository ); + assertTrue( repo.exists() ); + File indexDirectory = + new File( FileUtil.getBasedir(), "target/index/test-" + Long.toString( System.currentTimeMillis() ) ); + indexDirectory.deleteOnExit(); + FileUtils.deleteDirectory( indexDirectory ); context = nexusIndexer.addIndexingContext( repository, repository, repo, indexDirectory, repo.toURI().toURL().toExternalForm(), indexDirectory.toURI().toURL().toString(), search.getAllIndexCreators() ); - List artifactContexts = new ArrayList( filesToBeIndexed.size() ); + // minimize datas in memory + context.getIndexWriter().setMaxBufferedDocs( -1 ); + context.getIndexWriter().setRAMBufferSizeMB( 1 ); for ( File artifactFile : filesToBeIndexed ) { + assertTrue( "file not exists " + artifactFile.getPath(), artifactFile.exists() ); ArtifactContext ac = artifactContextProducer.getArtifactContext( context, artifactFile ); - artifactContexts.add( ac ); + nexusIndexer.addArtifactToIndex( ac, context ); + context.updateTimestamp( true ); } - if ( filesToBeIndexed != null && !filesToBeIndexed.isEmpty() ) - { - nexusIndexer.addArtifactsToIndex( artifactContexts, context ); - - while ( context.isReceivingUpdates() ) - { - Thread.sleep( 10 ); - } - } if ( scan ) { nexusIndexer.scan( context, new ArtifactScanListener(), false ); } - + // force flushing + context.getIndexWriter().commit(); + context.getIndexWriter().close( true ); + // wait for io flush .... + //Thread.sleep( 2000 ); context.setSearchable( true ); - assertTrue( new File( FileUtil.getBasedir(), "/target/repos/" + repository + "/.indexer" ).exists() ); } static class ArtifactScanListener diff --git a/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/NexusRepositorySearchTest.java b/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/NexusRepositorySearchTest.java index 90e2a5ec6..fb6f4ae0d 100644 --- a/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/NexusRepositorySearchTest.java +++ b/archiva-modules/archiva-base/archiva-indexer/src/test/java/org/apache/archiva/indexer/search/NexusRepositorySearchTest.java @@ -19,16 +19,14 @@ package org.apache.archiva.indexer.search; * under the License. */ +import org.apache.archiva.indexer.util.SearchUtil; import org.apache.maven.archiva.common.utils.FileUtil; -import org.apache.maven.index.artifact.IllegalArtifactCoordinateException; -import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -45,11 +43,11 @@ public class NexusRepositorySearchTest throws Exception { List files = new ArrayList(); - files.add( new File( FileUtil.getBasedir(), "src/test" + TEST_REPO_1 + files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 + "/org/apache/archiva/archiva-search/1.0/archiva-search-1.0.jar" ) ); files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 + "/org/apache/archiva/archiva-test/1.0/archiva-test-1.0.jar" ) ); - files.add( new File( FileUtil.getBasedir(), "/target/test-classes/" + TEST_REPO_1 + files.add( new File( FileUtil.getBasedir(), "/src/test/" + TEST_REPO_1 + "/org/apache/archiva/archiva-test/2.0/archiva-test-2.0.jar" ) ); createIndex( TEST_REPO_1, files, scan ); @@ -67,12 +65,12 @@ public class NexusRepositorySearchTest + "/org/apache/archiva/archiva-test/2.0/archiva-test-2.0.jar" ) ); files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 + "/org/apache/archiva/archiva-webapp/1.0/archiva-webapp-1.0.war" ) ); - files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 - + "/com/artifactid-numeric/1.0/artifactid-numeric-1.0.jar" ) ); + files.add( new File( FileUtil.getBasedir(), + "src/test/" + TEST_REPO_1 + "/com/artifactid-numeric/1.0/artifactid-numeric-1.0.jar" ) ); files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 + "/com/artifactid-numeric123/1.0/artifactid-numeric123-1.0.jar" ) ); - files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 - + "/com/classname-search/1.0/classname-search-1.0.jar" ) ); + files.add( new File( FileUtil.getBasedir(), + "src/test/" + TEST_REPO_1 + "/com/classname-search/1.0/classname-search-1.0.jar" ) ); createIndex( TEST_REPO_1, files, scan ); } @@ -108,9 +106,10 @@ public class NexusRepositorySearchTest archivaConfigControl.verify(); assertNotNull( results ); - assertEquals( 1, results.getTotalHits() ); - SearchResultHit hit = results.getHits().get( 0 ); + SearchResultHit hit = + results.getSearchResultHit( SearchUtil.getHitId( "org.apache.archiva", "archiva-search", null, "jar" ) ); + assertNotNull( "hit null in result " + results.getHits(), hit ); assertEquals( "org.apache.archiva", hit.getGroupId() ); assertEquals( "archiva-search", hit.getArtifactId() ); assertEquals( "1.0", hit.getVersions().get( 0 ) ); @@ -166,7 +165,7 @@ public class NexusRepositorySearchTest public void testMultipleArtifactsSameVersionWithClassifier() throws Exception { - createIndexContainingMultipleArtifactsSameVersion( false ); + createIndexContainingMultipleArtifactsSameVersion( true ); List selectedRepos = new ArrayList(); selectedRepos.add( TEST_REPO_1 ); @@ -291,17 +290,20 @@ public class NexusRepositorySearchTest archivaConfigControl.replay(); + // wait lucene flush..... + Thread.sleep( 2000 ); + SearchResults results = search.search( "user", selectedRepos, "archiva-search", null, null ); archivaConfigControl.verify(); assertNotNull( results ); - assertEquals( 4, results.getTotalHits() ); - SearchResultHit hit = results.getHits().get( 0 ); + SearchResultHit hit = + results.getSearchResultHit( SearchUtil.getHitId( "org.apache.archiva", "archiva-search", null, "jar" ) ); assertEquals( "org.apache.archiva", hit.getGroupId() ); assertEquals( "archiva-search", hit.getArtifactId() ); - assertEquals("not 2 version for hit " + hit, 2, hit.getVersions().size() ); + assertEquals( "not 2 version for hit " + hit + "::" + niceDisplay( results ), 2, hit.getVersions().size() ); assertTrue( hit.getVersions().contains( "1.0" ) ); assertTrue( hit.getVersions().contains( "1.1" ) ); @@ -489,8 +491,8 @@ public class NexusRepositorySearchTest throws Exception { List files = new ArrayList(); - files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 - + "/com/artifactid-numeric/1.0/artifactid-numeric-1.0.jar" ) ); + files.add( new File( FileUtil.getBasedir(), + "src/test/" + TEST_REPO_1 + "/com/artifactid-numeric/1.0/artifactid-numeric-1.0.jar" ) ); files.add( new File( FileUtil.getBasedir(), "src/test/" + TEST_REPO_1 + "/com/artifactid-numeric123/1.0/artifactid-numeric123-1.0.jar" ) ); createIndex( TEST_REPO_1, files, true ); @@ -688,8 +690,7 @@ public class NexusRepositorySearchTest { createIndexContainingMoreArtifacts( true ); - List selectedRepos = new ArrayList(); - selectedRepos.add( TEST_REPO_1 ); + List selectedRepos = Arrays.asList( TEST_REPO_1 ); SearchFields searchFields = new SearchFields(); searchFields.setClassName( "com.classname.search.App" ); diff --git a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java index 2477c7898..e36547142 100644 --- a/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java +++ b/archiva-modules/archiva-scheduler/archiva-scheduler-indexing/src/main/java/org/apache/archiva/scheduler/indexing/ArchivaIndexingTaskExecutor.java @@ -197,8 +197,6 @@ public class ArchivaIndexingTaskExecutor nexusIndexer.addArtifactToIndex( ac, context ); } - //nexusIndexer.scan( context, true ); - context.updateTimestamp(); // close the context if not a repo scan request