From 6b7feec1f102f11291d72c8c7b4ab50059ab5915 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Mon, 24 Mar 2014 01:36:39 +0000 Subject: [PATCH] fix checksum search cassandra search git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1580691 13f79535-47bb-0310-9956-ffa450edef68 --- .../repository/MetadataRepository.java | 2 +- .../AbstractMetadataRepositoryTest.java | 2 +- .../CassandraMetadataRepository.java | 126 ++++++++++-------- .../DefaultCassandraArchivaManager.java | 13 ++ .../consumers/DuplicateArtifactsConsumer.java | 3 +- 5 files changed, 86 insertions(+), 60 deletions(-) diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java index 7bec0d6bd..cdd865d56 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java +++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java @@ -98,7 +98,7 @@ public interface MetadataRepository Collection getRepositories() throws MetadataRepositoryException; - List getArtifactsByChecksum( String repositoryId, String checksum ) + Collection getArtifactsByChecksum( String repositoryId, String checksum ) throws MetadataRepositoryException; void removeArtifact( String repositoryId, String namespace, String project, String version, String id ) diff --git a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java index e11911671..c33c8d676 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java +++ b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java @@ -1164,7 +1164,7 @@ public abstract class AbstractMetadataRepositoryTest ArtifactMetadata artifact = createArtifact(); repository.updateArtifact( TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact ); - List artifactsByChecksum = repository.getArtifactsByChecksum( TEST_REPO_ID, "not checksum" ); + Collection artifactsByChecksum = repository.getArtifactsByChecksum( TEST_REPO_ID, "not checksum" ); assertEquals( Collections.emptyList(), artifactsByChecksum ); } diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java index 6eda35d5e..831686269 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java @@ -723,7 +723,7 @@ public class CassandraMetadataRepository return projectMetadata; } - protected ProjectVersionMetadataModel map( ColumnSlice columnSlice ) + protected ProjectVersionMetadataModel mapProjectVersionMetadataModel( ColumnSlice columnSlice ) { ProjectVersionMetadataModel projectVersionMetadataModel = new ProjectVersionMetadataModel(); projectVersionMetadataModel.setId( getStringValue( columnSlice, "id" ) ); @@ -786,7 +786,8 @@ public class CassandraMetadataRepository boolean creation = true; if ( result.get().getCount() > 0 ) { - projectVersionMetadataModel = map( result.get().getList().get( 0 ).getColumnSlice() ); + projectVersionMetadataModel = + mapProjectVersionMetadataModel( result.get().getList().get( 0 ).getColumnSlice() ); creation = false; } else @@ -1675,27 +1676,11 @@ public class CassandraMetadataRepository for ( Row row : result.get() ) { ColumnSlice columnSlice = row.getColumnSlice(); - String repositoryName = - ss.fromByteBuffer( columnSlice.getColumnByName( "repositoryName" ).getValueBytes() ); + String repositoryName = getAsStringValue( columnSlice, "repositoryName" ); if ( StringUtils.equals( repositoryName, repositoryId ) ) { - ArtifactMetadata artifactMetadata = new ArtifactMetadata(); - artifactMetadata.setNamespace( getAsStringValue( columnSlice, "namespaceId" ) ); - artifactMetadata.setSize( getLongValue( columnSlice, "size" ) ); - artifactMetadata.setId( getAsStringValue( columnSlice, "id" ) ); - artifactMetadata.setFileLastModified( getLongValue( columnSlice, "fileLastModified" ) ); - artifactMetadata.setMd5( getAsStringValue( columnSlice, "md5" ) ); - artifactMetadata.setProject( getAsStringValue( columnSlice, "project" ) ); - artifactMetadata.setProjectVersion( getAsStringValue( columnSlice, "projectVersion" ) ); - artifactMetadata.setRepositoryId( repositoryName ); - artifactMetadata.setSha1( getAsStringValue( columnSlice, "sha1" ) ); - artifactMetadata.setVersion( getAsStringValue( columnSlice, "version" ) ); - Long whenGathered = getLongValue( columnSlice, "whenGathered" ); - if ( whenGathered != null ) - { - artifactMetadata.setWhenGathered( new Date( whenGathered ) ); - } - artifactMetadatas.add( artifactMetadata ); + + artifactMetadatas.add( mapArtifactMetadataLongColumnSlice( columnSlice ) ); } } @@ -1703,6 +1688,27 @@ public class CassandraMetadataRepository } + protected ArtifactMetadata mapArtifactMetadataLongColumnSlice( ColumnSlice columnSlice ) + { + ArtifactMetadata artifactMetadata = new ArtifactMetadata(); + artifactMetadata.setNamespace( getAsStringValue( columnSlice, "namespaceId" ) ); + artifactMetadata.setSize( getLongValue( columnSlice, "size" ) ); + artifactMetadata.setId( getAsStringValue( columnSlice, "id" ) ); + artifactMetadata.setFileLastModified( getLongValue( columnSlice, "fileLastModified" ) ); + artifactMetadata.setMd5( getAsStringValue( columnSlice, "md5" ) ); + artifactMetadata.setProject( getAsStringValue( columnSlice, "project" ) ); + artifactMetadata.setProjectVersion( getAsStringValue( columnSlice, "projectVersion" ) ); + artifactMetadata.setRepositoryId( getStringValue( columnSlice, "repositoryName" ) ); + artifactMetadata.setSha1( getAsStringValue( columnSlice, "sha1" ) ); + artifactMetadata.setVersion( getAsStringValue( columnSlice, "version" ) ); + Long whenGathered = getLongValue( columnSlice, "whenGathered" ); + if ( whenGathered != null ) + { + artifactMetadata.setWhenGathered( new Date( whenGathered ) ); + } + return artifactMetadata; + } + protected void populateFacets( final ArtifactMetadata artifactMetadata ) { /* final List metadataFacetModels = new ArrayList(); @@ -1760,55 +1766,61 @@ public class CassandraMetadataRepository } @Override - public List getArtifactsByChecksum( final String repositoryId, final String checksum ) + public Collection getArtifactsByChecksum( final String repositoryId, final String checksum ) throws MetadataRepositoryException { -/* final List artifactMetadataModels = new ArrayList(); + Keyspace keyspace = cassandraArchivaManager.getKeyspace(); + StringSerializer ss = StringSerializer.get(); - if ( logger.isDebugEnabled() ) - { - logger.debug( "all ArtifactMetadataModel: {}", getArtifactMetadataModelEntityManager().getAll() ); - } + // cql cannot run or in queries so running twice the query + Map artifactMetadataMap = new HashMap(); - // FIXME cql query - getArtifactMetadataModelEntityManager().visitAll( new Function() + RangeSlicesQuery query = HFactory // + .createRangeSlicesQuery( keyspace, ss, ss, ss ) // + .setColumnFamily( cassandraArchivaManager.getArtifactMetadataModelFamilyName() ) // + .setColumnNames( "namespaceId", "size", "id", "fileLastModified", "md5", "project", "projectVersion", + "repositoryName", "version", "whenGathered", "sha1" ); // + + query = query.addEqualsExpression( "sha1", checksum ); + + QueryResult> result = query.execute(); + + for ( Row row : result.get() ) { - @Override - public Boolean apply( ArtifactMetadataModel artifactMetadataModel ) + ColumnSlice columnSlice = row.getColumnSlice(); + String repositoryName = getStringValue( columnSlice, "repositoryName" ); + if ( StringUtils.equals( repositoryName, repositoryId ) ) { - if ( artifactMetadataModel != null ) - { - if ( StringUtils.equals( artifactMetadataModel.getRepositoryId(), repositoryId ) - && artifactMetadataModel.getNamespace() != null && - artifactMetadataModel.getProject() != null && artifactMetadataModel.getId() != null ) - { - if ( StringUtils.equals( checksum, artifactMetadataModel.getMd5() ) || StringUtils.equals( - checksum, artifactMetadataModel.getSha1() ) ) - { - artifactMetadataModels.add( artifactMetadataModel ); - } - } - } - return Boolean.TRUE; + artifactMetadataMap.put( row.getKey(), mapArtifactMetadataLongColumnSlice( columnSlice ) ); } - } ); - List artifactMetadatas = new ArrayList( artifactMetadataModels.size() ); - - for ( ArtifactMetadataModel model : artifactMetadataModels ) - { - ArtifactMetadata artifactMetadata = getModelMapper().map( model, ArtifactMetadata.class ); - populateFacets( artifactMetadata ); - artifactMetadatas.add( artifactMetadata ); } - logger.debug( "getArtifactsByChecksum repositoryId: {}, checksum: {}, artifactMetadatas: {}", repositoryId, - checksum, artifactMetadatas ); + query = HFactory // + .createRangeSlicesQuery( keyspace, ss, ss, ss ) // + .setColumnFamily( cassandraArchivaManager.getArtifactMetadataModelFamilyName() ) // + .setColumnNames( "namespaceId", "size", "id", "fileLastModified", "md5", "project", "projectVersion", + "repositoryName", "version", "whenGathered", "sha1" ); // - return artifactMetadatas;*/ - return Collections.emptyList(); + query = query.addEqualsExpression( "md5", checksum ); + + result = query.execute(); + + for ( Row row : result.get() ) + { + ColumnSlice columnSlice = row.getColumnSlice(); + String repositoryName = getStringValue( columnSlice, "repositoryName" ); + if ( StringUtils.equals( repositoryName, repositoryId ) ) + { + + artifactMetadataMap.put( row.getKey(), mapArtifactMetadataLongColumnSlice( columnSlice ) ); + } + } + + return artifactMetadataMap.values(); } + @Override public void removeArtifact( final String repositoryId, final String namespace, final String project, final String version, final String id ) diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java index c993284d2..651507d02 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java @@ -288,6 +288,19 @@ public class DefaultCassandraArchivaManager whenGatheredColumn.setValidationClass( ComparatorType.LONGTYPE.getClassName() ); artifactMetadataModel.addColumnDefinition( whenGatheredColumn ); + BasicColumnDefinition sha1Column = new BasicColumnDefinition(); + sha1Column.setName( StringSerializer.get().toByteBuffer( "sha1" ) ); + sha1Column.setIndexName( "sha1" ); + sha1Column.setIndexType( ColumnIndexType.KEYS ); + sha1Column.setValidationClass( ComparatorType.UTF8TYPE.getClassName() ); + artifactMetadataModel.addColumnDefinition( sha1Column ); + + BasicColumnDefinition md5Column = new BasicColumnDefinition(); + md5Column.setName( StringSerializer.get().toByteBuffer( "md5" ) ); + md5Column.setIndexName( "md5" ); + md5Column.setIndexType( ColumnIndexType.KEYS ); + md5Column.setValidationClass( ComparatorType.UTF8TYPE.getClassName() ); + artifactMetadataModel.addColumnDefinition( md5Column ); } diff --git a/archiva-modules/plugins/problem-reports/src/main/java/org/apache/archiva/reports/consumers/DuplicateArtifactsConsumer.java b/archiva-modules/plugins/problem-reports/src/main/java/org/apache/archiva/reports/consumers/DuplicateArtifactsConsumer.java index 73ff6e42f..e4cd0a902 100644 --- a/archiva-modules/plugins/problem-reports/src/main/java/org/apache/archiva/reports/consumers/DuplicateArtifactsConsumer.java +++ b/archiva-modules/plugins/problem-reports/src/main/java/org/apache/archiva/reports/consumers/DuplicateArtifactsConsumer.java @@ -49,6 +49,7 @@ import javax.inject.Named; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; @@ -158,7 +159,7 @@ public class DuplicateArtifactsConsumer MetadataRepository metadataRepository = repositorySession.getRepository(); - List results; + Collection results; try { results = metadataRepository.getArtifactsByChecksum( repoId, checksumSha1 );