diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index 544b62275f9..76189905334 100644
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -249,8 +249,12 @@ def checkJARMetaData(desc, jarFile, svnRevision, version):
def normSlashes(path):
return path.replace(os.sep, '/')
-def checkAllJARs(topDir, project, svnRevision, version):
- print(' verify JAR/WAR metadata...')
+def checkAllJARs(topDir, project, svnRevision, version, tmpDir, baseURL):
+ print(' verify JAR metadata/identity/no javax.* or java.* classes...')
+ if project == 'solr':
+ luceneDistFilenames = dict()
+ for file in getBinaryDistFiles('lucene', tmpDir, version, baseURL):
+ luceneDistFilenames[os.path.basename(file)] = file
for root, dirs, files in os.walk(topDir):
normRoot = normSlashes(root)
@@ -272,29 +276,54 @@ def checkAllJARs(topDir, project, svnRevision, version):
noJavaPackageClasses('JAR file "%s"' % fullPath, fullPath)
if file.lower().find('lucene') != -1 or file.lower().find('solr') != -1:
checkJARMetaData('JAR file "%s"' % fullPath, fullPath, svnRevision, version)
-
+ if project == 'solr' and file.lower().find('lucene') != -1:
+ jarFilename = os.path.basename(file)
+ if jarFilename not in luceneDistFilenames:
+ raise RuntimeError('Artifact %s is not present in Lucene binary distribution' % fullPath)
+ identical = filecmp.cmp(fullPath, luceneDistFilenames[jarFilename], shallow=False)
+ if not identical:
+ raise RuntimeError('Artifact %s is not identical to %s in Lucene binary distribution'
+ % (fullPath, luceneDistFilenames[jarFilename]))
-def checkSolrWAR(warFileName, svnRevision, version):
+
+def checkSolrWAR(warFileName, svnRevision, version, tmpDir, baseURL):
"""
Crawls for JARs inside the WAR and ensures there are no classes
under java.* or javax.* namespace.
"""
- print(' make sure WAR file has no javax.* or java.* classes...')
+ print(' verify WAR metadata/contained JAR identity/no javax.* or java.* classes...')
checkJARMetaData(warFileName, warFileName, svnRevision, version)
+ distFilenames = dict()
+ for file in getBinaryDistFiles('lucene', tmpDir, version, baseURL):
+ distFilenames[os.path.basename(file)] = file
+
with zipfile.ZipFile(warFileName, 'r') as z:
for name in z.namelist():
if name.endswith('.jar'):
+ jarInsideWarContents = z.read(name)
noJavaPackageClasses('JAR file %s inside WAR file %s' % (name, warFileName),
- io.BytesIO(z.read(name)))
+ io.BytesIO(jarInsideWarContents))
if name.lower().find('lucene') != -1 or name.lower().find('solr') != -1:
checkJARMetaData('JAR file %s inside WAR file %s' % (name, warFileName),
- io.BytesIO(z.read(name)),
+ io.BytesIO(jarInsideWarContents),
svnRevision,
version)
+ if name.lower().find('lucene') != -1:
+ jarInsideWarFilename = os.path.basename(name)
+ if jarInsideWarFilename not in distFilenames:
+ raise RuntimeError('Artifact %s in %s is not present in Lucene binary distribution'
+ % (name, warFileName))
+ distJarName = distFilenames[jarInsideWarFilename]
+ with open(distJarName, "rb", buffering=0) as distJarFile:
+ distJarContents = distJarFile.readall()
+ if jarInsideWarContents != distJarContents:
+ raise RuntimeError('Artifact %s in %s is not identical to %s in Lucene binary distribution'
+ % (name, warFileName, distJarName))
+
def checkSigs(project, urlString, version, tmpDir, isSigned):
@@ -575,7 +604,7 @@ def getDirEntries(urlString):
if text == 'Parent Directory' or text == '..':
return links[(i+1):]
-def unpackAndVerify(project, tmpDir, artifact, svnRevision, version, testArgs):
+def unpackAndVerify(project, tmpDir, artifact, svnRevision, version, testArgs, baseURL):
destDir = '%s/unpack' % tmpDir
if os.path.exists(destDir):
shutil.rmtree(destDir)
@@ -595,14 +624,14 @@ def unpackAndVerify(project, tmpDir, artifact, svnRevision, version, testArgs):
raise RuntimeError('unpack produced entries %s; expected only %s' % (l, expected))
unpackPath = '%s/%s' % (destDir, expected)
- verifyUnpacked(project, artifact, unpackPath, svnRevision, version, testArgs)
+ verifyUnpacked(project, artifact, unpackPath, svnRevision, version, testArgs, tmpDir, baseURL)
LUCENE_NOTICE = None
LUCENE_LICENSE = None
SOLR_NOTICE = None
SOLR_LICENSE = None
-def verifyUnpacked(project, artifact, unpackPath, svnRevision, version, testArgs):
+def verifyUnpacked(project, artifact, unpackPath, svnRevision, version, testArgs, tmpDir, baseURL):
global LUCENE_NOTICE
global LUCENE_LICENSE
global SOLR_NOTICE
@@ -720,13 +749,13 @@ def verifyUnpacked(project, artifact, unpackPath, svnRevision, version, testArgs
else:
- checkAllJARs(os.getcwd(), project, svnRevision, version)
+ checkAllJARs(os.getcwd(), project, svnRevision, version, tmpDir, baseURL)
if project == 'lucene':
testDemo(isSrc, version, '1.7')
else:
- checkSolrWAR('%s/example/webapps/solr.war' % unpackPath, svnRevision, version)
+ checkSolrWAR('%s/example/webapps/solr.war' % unpackPath, svnRevision, version, tmpDir, baseURL)
print(' copying unpacked distribution for Java 7 ...')
java7UnpackPath = '%s-java7' %unpackPath
@@ -913,11 +942,10 @@ def checkMaven(baseURL, tmpDir, svnRevision, version, isSigned):
if text == releaseBranchText:
releaseBranchSvnURL = subURL
- print(' get POM templates', end=' ')
POMtemplates = defaultdict()
getPOMtemplates(POMtemplates, tmpDir, releaseBranchSvnURL)
print()
- print(' download artifacts', end=' ')
+ print(' download artifacts')
artifacts = {'lucene': [], 'solr': []}
for project in ('lucene', 'solr'):
artifactsURL = '%s/%s/maven/org/apache/%s' % (baseURL, project, project)
@@ -926,50 +954,47 @@ def checkMaven(baseURL, tmpDir, svnRevision, version, isSigned):
os.makedirs(targetDir)
crawl(artifacts[project], artifactsURL, targetDir)
print()
- print(' verify that each binary artifact has a deployed POM...')
verifyPOMperBinaryArtifact(artifacts, version)
- print(' verify that there is an artifact for each POM template...')
verifyArtifactPerPOMtemplate(POMtemplates, artifacts, tmpDir, version)
- print(" verify Maven artifacts' md5/sha1 digests...")
verifyMavenDigests(artifacts)
- print(' check for javadoc and sources artifacts...')
checkJavadocAndSourceArtifacts(artifacts, version)
- print(" verify deployed POMs' coordinates...")
verifyDeployedPOMsCoordinates(artifacts, version)
if isSigned:
- print(' verify maven artifact sigs', end=' ')
verifyMavenSigs(baseURL, tmpDir, artifacts)
- distributionFiles = getDistributionsForMavenChecks(tmpDir, version, baseURL)
+ distFiles = getBinaryDistFilesForMavenChecks(tmpDir, version, baseURL)
+ checkIdenticalMavenArtifacts(distFiles, artifacts, version)
- print(' verify that Maven artifacts are same as in the binary distribution...')
- checkIdenticalMavenArtifacts(distributionFiles, artifacts, version)
+ checkAllJARs('%s/maven/org/apache/lucene' % tmpDir, 'lucene', svnRevision, version, tmpDir, baseURL)
+ checkAllJARs('%s/maven/org/apache/solr' % tmpDir, 'solr', svnRevision, version, tmpDir, baseURL)
- checkAllJARs('%s/maven/org/apache/lucene' % tmpDir, 'lucene', svnRevision, version)
- checkAllJARs('%s/maven/org/apache/solr' % tmpDir, 'solr', svnRevision, version)
-
-def getDistributionsForMavenChecks(tmpDir, version, baseURL):
- distributionFiles = defaultdict()
+def getBinaryDistFilesForMavenChecks(tmpDir, version, baseURL):
+ distFiles = defaultdict()
for project in ('lucene', 'solr'):
- distribution = '%s-%s.tgz' % (project, version)
- if not os.path.exists('%s/%s' % (tmpDir, distribution)):
- distURL = '%s/%s/%s' % (baseURL, project, distribution)
- print(' download %s...' % distribution, end=' ')
- download(distribution, distURL, tmpDir)
- destDir = '%s/unpack-%s-maven' % (tmpDir, project)
- if os.path.exists(destDir):
- shutil.rmtree(destDir)
- os.makedirs(destDir)
- os.chdir(destDir)
- print(' unpack %s...' % distribution)
- unpackLogFile = '%s/unpack-%s-maven-checks.log' % (tmpDir, distribution)
- run('tar xzf %s/%s' % (tmpDir, distribution), unpackLogFile)
- distributionFiles[project] = []
- for root, dirs, files in os.walk(destDir):
- distributionFiles[project].extend([os.path.join(root, file) for file in files])
+ distFiles[project] = getBinaryDistFiles(project, tmpDir, version, baseURL)
+ return distFiles
+
+def getBinaryDistFiles(project, tmpDir, version, baseURL):
+ distribution = '%s-%s.tgz' % (project, version)
+ if not os.path.exists('%s/%s' % (tmpDir, distribution)):
+ distURL = '%s/%s/%s' % (baseURL, project, distribution)
+ print(' download %s...' % distribution, end=' ')
+ download(distribution, distURL, tmpDir)
+ destDir = '%s/unpack-%s-getBinaryDistFiles' % (tmpDir, project)
+ if os.path.exists(destDir):
+ shutil.rmtree(destDir)
+ os.makedirs(destDir)
+ os.chdir(destDir)
+ print(' unpack %s...' % distribution)
+ unpackLogFile = '%s/unpack-%s-getBinaryDistFiles.log' % (tmpDir, distribution)
+ run('tar xzf %s/%s' % (tmpDir, distribution), unpackLogFile)
+ distributionFiles = []
+ for root, dirs, files in os.walk(destDir):
+ distributionFiles.extend([os.path.join(root, file) for file in files])
return distributionFiles
def checkJavadocAndSourceArtifacts(artifacts, version):
+ print(' check for javadoc and sources artifacts...')
for project in ('lucene', 'solr'):
for artifact in artifacts[project]:
if artifact.endswith(version + '.jar'):
@@ -989,35 +1014,28 @@ def getZipFileEntries(fileName):
entries.sort()
return entries
-def checkIdenticalMavenArtifacts(distributionFiles, artifacts, version):
+def checkIdenticalMavenArtifacts(distFiles, artifacts, version):
+ print(' verify that Maven artifacts are same as in the binary distribution...')
reJarWar = re.compile(r'%s\.[wj]ar$' % version) # exclude *-javadoc.jar and *-sources.jar
for project in ('lucene', 'solr'):
distFilenames = dict()
- for file in distributionFiles[project]:
+ for file in distFiles[project]:
baseName = os.path.basename(file)
distFilenames[baseName] = file
for artifact in artifacts[project]:
if reJarWar.search(artifact):
- entries = getZipFileEntries(artifact)
artifactFilename = os.path.basename(artifact)
if artifactFilename not in distFilenames:
raise RuntimeError('Maven artifact %s is not present in %s binary distribution'
% (artifact, project))
else:
- binaryEntries = getZipFileEntries(distFilenames[artifactFilename])
- if binaryEntries != entries:
- raise RuntimeError('Maven artifact %s has different contents than binary distribution\n maven:\n%s\n binary:\n%s\n' % \
- (artifactFilename,
- '\n'.join(entries),
- '\n'.join(binaryEntries)))
-
- # TODO: Either fix the build to ensure that maven artifacts *are* identical, or recursively compare contents
- # identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
- # if not identical:
- # raise RuntimeError('Maven artifact %s is not identical to %s in %s binary distribution'
- # % (artifact, distFilenames[artifactFilename], project))
+ identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
+ if not identical:
+ raise RuntimeError('Maven artifact %s is not identical to %s in %s binary distribution'
+ % (artifact, distFilenames[artifactFilename], project))
def verifyMavenDigests(artifacts):
+ print(" verify Maven artifacts' md5/sha1 digests...")
reJarWarPom = re.compile(r'\.(?:[wj]ar|pom)$')
for project in ('lucene', 'solr'):
for artifactFile in [a for a in artifacts[project] if reJarWarPom.search(a)]:
@@ -1064,7 +1082,7 @@ def getPOMcoordinate(treeRoot):
return groupId, artifactId, packaging, version
def verifyMavenSigs(baseURL, tmpDir, artifacts):
- """Verify Maven artifact signatures"""
+ print(' verify maven artifact sigs', end=' ')
for project in ('lucene', 'solr'):
keysFile = '%s/%s.KEYS' % (tmpDir, project)
if not os.path.exists(keysFile):
@@ -1114,7 +1132,7 @@ def verifyMavenSigs(baseURL, tmpDir, artifacts):
print()
def verifyPOMperBinaryArtifact(artifacts, version):
- """verify that each binary jar and war has a corresponding POM file"""
+ print(' verify that each binary artifact has a deployed POM...')
reBinaryJarWar = re.compile(r'%s\.[jw]ar$' % re.escape(version))
for project in ('lucene', 'solr'):
for artifact in [a for a in artifacts[project] if reBinaryJarWar.search(a)]:
@@ -1127,6 +1145,7 @@ def verifyDeployedPOMsCoordinates(artifacts, version):
verify that each POM's coordinate (drawn from its content) matches
its filepath, and verify that the corresponding artifact exists.
"""
+ print(" verify deployed POMs' coordinates...")
for project in ('lucene', 'solr'):
for POM in [a for a in artifacts[project] if a.endswith('.pom')]:
treeRoot = ET.parse(POM).getroot()
@@ -1142,7 +1161,7 @@ def verifyDeployedPOMsCoordinates(artifacts, version):
raise RuntimeError('Missing corresponding .%s artifact for POM %s' % (packaging, POM))
def verifyArtifactPerPOMtemplate(POMtemplates, artifacts, tmpDir, version):
- """verify that each POM template's artifact is present in artifacts"""
+ print(' verify that there is an artifact for each POM template...')
namespace = '{http://maven.apache.org/POM/4.0.0}'
xpathPlugin = '{0}build/{0}plugins/{0}plugin'.format(namespace)
xpathSkipConfiguration = '{0}configuration/{0}skip'.format(namespace)
@@ -1165,6 +1184,7 @@ def verifyArtifactPerPOMtemplate(POMtemplates, artifacts, tmpDir, version):
raise RuntimeError('Missing artifact %s' % artifact)
def getPOMtemplates(POMtemplates, tmpDir, releaseBranchSvnURL):
+ print(' get POM templates')
allPOMtemplates = []
sourceLocation = releaseBranchSvnURL
if sourceLocation is None:
@@ -1292,15 +1312,15 @@ def smokeTest(baseURL, svnRevision, version, tmpDir, isSigned, testArgs):
print('Test Lucene...')
checkSigs('lucene', lucenePath, version, tmpDir, isSigned)
for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
- unpackAndVerify('lucene', tmpDir, artifact, svnRevision, version, testArgs)
- unpackAndVerify('lucene', tmpDir, 'lucene-%s-src.tgz' % version, svnRevision, version, testArgs)
+ unpackAndVerify('lucene', tmpDir, artifact, svnRevision, version, testArgs, baseURL)
+ unpackAndVerify('lucene', tmpDir, 'lucene-%s-src.tgz' % version, svnRevision, version, testArgs, baseURL)
print()
print('Test Solr...')
checkSigs('solr', solrPath, version, tmpDir, isSigned)
for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
- unpackAndVerify('solr', tmpDir, artifact, svnRevision, version, testArgs)
- unpackAndVerify('solr', tmpDir, 'solr-%s-src.tgz' % version, svnRevision, version, testArgs)
+ unpackAndVerify('solr', tmpDir, artifact, svnRevision, version, testArgs, baseURL)
+ unpackAndVerify('solr', tmpDir, 'solr-%s-src.tgz' % version, svnRevision, version, testArgs, baseURL)
print()
print('Test Maven artifacts for Lucene and Solr...')
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0663f433681..aa4248d415b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -173,6 +173,10 @@ Build
* LUCENE-5249, LUCENE-5257: All Lucene/Solr modules should use the same
dependency versions. (Steve Rowe)
+
+* LUCENE-5273: Binary artifacts in Lucene and Solr convenience binary
+ distributions accompanying a release, including on Maven Central,
+ should be identical across all distributions. (Steve Rowe)
======================= Lucene 4.5.0 =======================
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index ce46e333893..ddbc5a7ee95 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -490,12 +490,35 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -1353,9 +1376,9 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites
+ depends="filter-pom-templates, install-maven-tasks, m2-deploy-lucene-parent-pom, -unpack-lucene-tgz, dist-maven-common"/>
+ depends="jar-src, javadocs, install-maven-tasks, filter-pom-templates">
@@ -1379,9 +1402,9 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites
+ depends="filter-pom-templates, install-maven-tasks, m2-deploy-lucene-parent-pom, -unpack-lucene-tgz, dist-maven-common-src-java"/>
+ depends="-unpack-lucene-tgz, jar-src, javadocs, install-maven-tasks, filter-pom-templates">
diff --git a/solr/common-build.xml b/solr/common-build.xml
index ce6cc1e247a..f728e7fc291 100644
--- a/solr/common-build.xml
+++ b/solr/common-build.xml
@@ -44,8 +44,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -142,19 +158,17 @@
-
-
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
@@ -309,10 +323,10 @@
+ depends="filter-pom-templates, install-maven-tasks, m2-deploy-solr-parent-pom, -unpack-solr-tgz, dist-maven-common"/>
+ depends="filter-pom-templates, install-maven-tasks, m2-deploy-solr-parent-pom, -unpack-solr-tgz, dist-maven-common-src-java"/>
diff --git a/solr/contrib/analysis-extras/build.xml b/solr/contrib/analysis-extras/build.xml
index 2bee5c584a8..666abab9ead 100644
--- a/solr/contrib/analysis-extras/build.xml
+++ b/solr/contrib/analysis-extras/build.xml
@@ -49,11 +49,14 @@
code in the analysis-extras contrib, they must remain here in order to
populate the Solr distribution
-->
-
+
+
+
+
+
-
+
diff --git a/solr/contrib/uima/build.xml b/solr/contrib/uima/build.xml
index 575b1bc79f7..22a06301c20 100644
--- a/solr/contrib/uima/build.xml
+++ b/solr/contrib/uima/build.xml
@@ -26,15 +26,23 @@
-
+
+
+
+
+
-
+
+
+
+
+
-
+
diff --git a/solr/test-framework/build.xml b/solr/test-framework/build.xml
index 61036faf1f4..b117d8b0242 100644
--- a/solr/test-framework/build.xml
+++ b/solr/test-framework/build.xml
@@ -71,11 +71,14 @@
-
+
+
+
+
+
-
+
diff --git a/solr/webapp/build.xml b/solr/webapp/build.xml
index 6a519b7143b..e98e036c817 100644
--- a/solr/webapp/build.xml
+++ b/solr/webapp/build.xml
@@ -42,7 +42,7 @@
+ depends="test, init-dist, dist-core, dist-solrj, -unpack-lucene-tgz, lucene-jars-to-solr">