LUCENE-10174 Update buildAndPushRelease.py for new gradle build (#382)

Co-authored-by: Tomoko Uchida <tomoko.uchida.1111@gmail.com>

dev-tools/scripts/buildAndPushRelease.py

@@ -30,6 +30,7 @@ import xml.etree.ElementTree as ET
 import scriptutil
 
 LOG = '/tmp/release.log'
+dev_mode = False
 
 def log(msg):
   f = open(LOG, mode='ab')
@@ -43,6 +44,7 @@ def run(command):
     print(msg)
     raise RuntimeError(msg)
 
+
 def runAndSendGPGPassword(command, password):
   p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
   f = open(LOG, 'ab')
@@ -77,6 +79,7 @@ def load(urlString, encoding="utf-8"):
   return content
 
 def getGitRev():
+  if not dev_mode:
     status = os.popen('git status').read().strip()
     if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status:
       raise RuntimeError('git clone is dirty:\n\n%s' % status)
@@ -90,38 +93,63 @@ def getGitRev():
       raise RuntimeError('Command "%s" failed:\n\n%s' % (command, stderr.decode('utf-8')))
 
     print('  git clone is clean')
+  else:
+    print('  Ignoring dirty git clone due to dev-mode')
   return os.popen('git rev-parse HEAD').read().strip()
 
-def prepare(root, version, gpgKeyID, gpgPassword):
+
+def prepare(root, version, gpg_key_id, gpg_password, gpg_home=None, sign_gradle=False):
   print()
   print('Prepare release...')
   if os.path.exists(LOG):
     os.remove(LOG)
 
+  if not dev_mode:
     os.chdir(root)
     print('  git pull...')
     run('git pull')
+  else:
+    print('  Development mode, not running git pull')
 
   rev = getGitRev()
   print('  git rev: %s' % rev)
   log('\nGIT rev: %s\n' % rev)
+  with open('rev.txt', mode='wb') as f:
+      f.write(rev.encode('UTF-8'))
 
   print('  Check DOAP files')
   checkDOAPfiles(version)
 
+  if not dev_mode:
     print('  ./gradlew -Dtests.badapples=false clean check')
     run('./gradlew -Dtests.badapples=false clean check')
-
+  else:
-  open('rev.txt', mode='wb').write(rev.encode('UTF-8'))
+    print('  skipping precommit check due to dev-mode')
 
   print('  prepare-release')
-  cmd = './gradlew -Dversion.release=%s clean assembleDist' % version
+  cmd = './gradlew clean assembleRelease' \
-  if gpgKeyID is not None:
+        ' -Dversion.release=%s' % version
-    cmd += ' -Psigning.gnupg.keyName=%s signDist' % gpgKeyID
+  if dev_mode:
-  cmd += ' mavenToLocalFolder'
+    cmd += ' -Pvalidation.git.failOnModified=false'
+  if gpg_key_id is not None:
+    cmd += ' -Psign'
+    if sign_gradle:
+      print("  Signing method is gradle java-plugin")
+      cmd += ' -Psigning.keyId="%s"' % gpg_key_id
+      if gpg_home is not None:
+        cmd += ' -Psigning.secretKeyRingFile="%s"' % os.path.join(gpg_home, 'secring.gpg')
+      if gpg_password is not None:
+        # Pass gpg passphrase as env.var to gradle rather than as plaintext argument
+        os.environ['ORG_GRADLE_PROJECT_signingPassword'] = gpg_password
+    else:
+      print("  Signing method is gpg tool")
+      cmd += ' -PuseGpg -Psigning.gnupg.keyName="%s"' % gpg_key_id
+      if gpg_home is not None:
+        cmd += ' -Psigning.gnupg.homeDir="%s"' % gpg_home
 
-  if gpgPassword is not None:
+  print("  Running: %s" % cmd)
-    runAndSendGPGPassword(cmd, gpgPassword)
+  if gpg_password is not None:
+    runAndSendGPGPassword(cmd, gpg_password)
   else:
     run(cmd)
 
@@ -180,6 +208,7 @@ def normalizeVersion(tup):
     tup = tup + ('0',)
   return '.'.join(tup) + suffix
 
+
 def pushLocal(version, root, rev, rcNum, localDir):
   print('Push local [%s]...' % localDir)
   os.makedirs(localDir)
@@ -187,7 +216,7 @@ def pushLocal(version, root, rev, rcNum, localDir):
   dir = 'lucene-%s-RC%d-rev%s' % (version, rcNum, rev)
   os.makedirs('%s/%s/lucene' % (localDir, dir))
   print('  Lucene')
-  lucene_dist_dir = '%s/lucene/packaging/build/distributions' % root
+  lucene_dist_dir = '%s/lucene/distribution/build/release' % root
   os.chdir(lucene_dist_dir)
   print('    zip...')
   if os.path.exists('lucene.tar.bz2'):
@@ -206,9 +235,11 @@ def pushLocal(version, root, rev, rcNum, localDir):
   print('  done!')
   return 'file://%s/%s' % (os.path.abspath(localDir), dir)
 
+
 def read_version(path):
   return scriptutil.find_current_version()
 
+
 def parse_config():
   epilogue = textwrap.dedent('''
     Example usage for a Release Manager:
@@ -225,12 +256,24 @@ def parse_config():
                       help='Push the release to the local path')
   parser.add_argument('--sign', metavar='KEYID',
                       help='Sign the release with the given gpg key')
+  parser.add_argument('--sign-method-gradle', dest='sign_method_gradle', default=False, action='store_true',
+                      help='Use Gradle built-in GPG signing instead of gpg command for signing artifacts. '
+                      ' This may require --gpg-secring argument if your keychain cannot be resolved automatically.')
+  parser.add_argument('--gpg-pass-noprompt', dest='gpg_pass_noprompt', default=False, action='store_true',
+                      help='Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent'
+                      ' needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided'
+                      ' in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info')
+  parser.add_argument('--gpg-home', metavar='PATH',
+                      help='Path to gpg home containing your secring.gpg'
+                      ' Optional, will use $HOME/.gnupg/secring.gpg by default')
   parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
                       help='Release Candidate number.  Default: 1')
   parser.add_argument('--root', metavar='PATH', default='.',
                       help='Root of Git working tree for lucene.  Default: "." (the current directory)')
   parser.add_argument('--logfile', metavar='PATH',
                       help='Specify log file path (default /tmp/release.log)')
+  parser.add_argument('--dev-mode', default=False, action='store_true',
+                      help='Enable development mode, which disables some strict checks')
   config = parser.parse_args()
 
   if not config.prepare and config.sign:
@@ -243,6 +286,12 @@ def parse_config():
     parser.error('Root path "%s" is not a directory' % config.root)
   if config.local_keys is not None and not os.path.exists(config.local_keys):
     parser.error('Local KEYS file "%s" not found' % config.local_keys)
+  if config.gpg_home and not os.path.exists(os.path.join(config.gpg_home, 'secring.gpg')):
+    parser.error('Specified gpg home %s does not exist or does not contain a secring.gpg' % config.gpg_home)
+  global dev_mode
+  if config.dev_mode:
+    print("Enabling development mode - DO NOT USE FOR ACTUAL RELEASE!")
+    dev_mode = True
   cwd = os.getcwd()
   os.chdir(config.root)
   config.root = os.getcwd() # Absolutize root dir
@@ -252,6 +301,7 @@ def parse_config():
   global LOG
   if config.logfile:
     LOG = config.logfile
+  print("Logfile is: %s" % LOG)
 
   config.version = read_version(config.root)
   print('Building version: %s' % config.version)
@@ -298,15 +348,47 @@ def check_key_in_keys(gpgKeyID, local_keys):
       exit(2)
 
 
+def resolve_gpghome():
+  for p in [
+    # Linux, macos
+    os.path.join(os.path.expanduser("~"), '.gnupg'),
+    # Windows 10
+    os.path.expandvars(r'%APPDATA%\GnuPG')
+    # TODO: Should we support Cygwin?
+  ]:
+    if os.path.exists(os.path.join(p, 'secring.gpg')):
+      return p
+  return None
+
+
 def main():
   check_cmdline_tools()
 
   c = parse_config()
+  gpg_home = None
 
   if c.sign:
     sys.stdout.flush()
     c.key_id = c.sign
     check_key_in_keys(c.key_id, c.local_keys)
+    if c.gpg_home is not None:
+      print("Using custom gpg-home: %s" % c.gpg_home)
+      gpg_home = c.gpg_home
+    if c.sign_method_gradle:
+      if gpg_home is None:
+        resolved_gpg_home = resolve_gpghome()
+        if resolved_gpg_home is not None:
+          print("Resolved gpg home to %s" % resolved_gpg_home)
+          gpg_home = resolved_gpg_home
+        else:
+          print("WARN: Could not locate your gpg secret keyring, and --gpg-home not specified.")
+          print("      Falling back to location configured in gradle.properties.")
+          print("      See 'gradlew helpPublishing' for details.")
+          gpg_home = None
+    if c.gpg_pass_noprompt:
+      print("Will not prompt for gpg password. Make sure your signing setup supports this.")
+      c.key_password = None
+    else:
       import getpass
       c.key_password = getpass.getpass('Enter GPG keystore password: ')
   else:
@@ -314,7 +396,7 @@ def main():
     c.key_password = None
 
   if c.prepare:
-    rev = prepare(c.root, c.version, c.key_id, c.key_password)
+    rev = prepare(c.root, c.version, c.key_id, c.key_password, gpg_home=gpg_home, sign_gradle=c.sign_method_gradle)
   else:
     os.chdir(c.root)
     rev = open('rev.txt', encoding='UTF-8').read()