lucene/dev-tools/scripts/svnBranchToGit.py

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Workaround for slow updates from an svn branch to git.
See also jira issue INFRA-9182

Situation:

Remote svn repo        ---> (slow) git-svn fetch  --->   Remote git repo (upstream)
   |                                                        |
   |                                                        |
   v                                                        v
Local svn working copy ---> this workaround       --->   Local git repo

When the remote git-svn fetch is slow, the remote git repo is behind
the remote svn repo.

When this script is run it will first check that the local working copy and repository are clean.
Then it switches the svn working copy to the branch, which updates from the remote.
Then it the fetches branch from the git upstream repo, and merges the branch locally.
Normally the local svn and git will then be at the same svn revision, and the script will exit.

Otherwise the remote git repo is out of date, and the following happens.

For the branch branchname in a local git repository following an upstream git-svn git repository,
this maintains commits on a temporary git branch branchname.svn in the local git repository.
These commits contain a message ending like this:
  "SvnRepoUrl diff -r EarlierSvnRevisionNumber:NextSvnRevisionNumber".
Otherwise the messages of the added commits are the same as their counterparts from git svn.

Normally the added git commits and their git-svn counterparts have no differences between their working trees.
However such differences can occur, see also the documentation of git-svn reset and the limitations below.
In order not to interfere with git-svn this script only adds commits to a temporary branch
branchname.svn, and the commit messages are chosen differently, they do not contain git-svn-id.

In case an earlier branchname.svn exists, it will first be deleted if necessary,
and restarted at the later branch.
Therefore branchname.svn is temporary and should only be used locally.

By default, no more than 20 commits will be added to branchname.svn in a single run.

The earlier revision number is taken from the git-svn-id message of git svn,
or from the latest revision number in the commit message on branchname.svn,
whichever is later.

This allows branchname.svn to be used as a local git branch instead of branchname
to develop new features locally, for example by merging branchname.svn into a feature branch.
"""

""" Limitations:

This currently works by patching text, and therefore this does not work on binary files.
An example commit in lucene-solr that adds a binary file, on which this currently does not work correctly:
svn revision 1707457
git commit 3c0390f71e1f08a17f32bc207b4003362f8b6ac2

When the local svn working copy contains file after updating to the latest available revision,
and there is an interim commit that deletes this file, this file is left as an empty file in the working directory
of the local git repository.

All svn properties are ignored here.
"""

""" To be done:
Take binary files from the patch, and check out binary files directly from the remote svn repo directly into the local git repo.

Going really far: checkout each interim svn revision, and use all (changed) files from there instead of the text diff.
Determining all files under version control with svn (svn ls) is far too slow for this (esp. when compared to git ls-tree),
so this is probably better done by using svnsync to setup a local mirror repository following the remote,
and then using svnlook on the local mirror repository.
Doing that only to avoid the limitations of this workaround does not appear to be worthwhile.
"""

""" This was developed on Linux using the following program versions:
python 2.7.6
git 1.9.1
svn 1.8.8
grep (GNU grep) 2.16

gitk (part of git) was used for manual testing:
- reset branch to an earlier commit to simulate a non working update from svn to git,
- delete branchname.svn, reset branchname.svn to earlier,
- diff a commit generated here to a commit from git svn,
- update, reload, show commits by commit date, ...
"""

import os
import subprocess

from xml import sax
from xml.sax.handler import ContentHandler

import types

class SvnInfoHandler(ContentHandler):
  revisionAttr = "revision"

  def __init__(self):
    self.lastChangeRev = None

  def startElement(self, name, attrs):
    if name == "commit":
      self.lastChangeRev = int(attrs.getValue(self.revisionAttr))

  def getLastChangeRevision(self):
    return self.lastChangeRev


class SvnLogEntry(object):
  pass # attributes set in SvnLogHandler: revision, author, date, msg


class SvnLogHandler(ContentHandler): # collect list of SvnLogEntry's
  logEntryTag = "logentry"
  revisionAttr = "revision" # also used as SvnLogEntry attribute
  authorTag = "author"
  dateTag = "date"
  msgTag = "msg"
  charCollectTags = (authorTag, dateTag, msgTag) # also used as SvnLogEntry attributes

  def __init__(self):
    self.logEntries = []
    self.chars = None

  def startElement(self, name, attrs):
    if name == self.logEntryTag:
      self.lastLogEntry = SvnLogEntry()
      setattr(self.lastLogEntry, self.revisionAttr, int(attrs.getValue(self.revisionAttr)))
      for tag in self.charCollectTags:
        setattr(self.lastLogEntry, tag, None)
      return

    if name in self.charCollectTags:
      self.chars = ""

  def characters(self, content):
    if self.chars is not None:
      self.chars += content

  def endElement(self, name):
    if name in self.charCollectTags:
      setattr(self.lastLogEntry, name, self.chars)
      self.chars = None
      return

    if name == self.logEntryTag:
      self.logEntries.append(self.lastLogEntry)
      self.lastLogEntry = None

  def getLogEntries(self):
    return self.logEntries


class SubProcessAtPath(object):
  def __init__(self, pathName, verbose=True):
    assert pathName != ""
    self.pathName = pathName
    self.verbose = verbose

  def getPathName(self):
    return self.pathName

  def chDirToPath(self):
    if self.pathName != os.getcwd():
      os.chdir(self.pathName)
      assert self.pathName == os.getcwd()

  def __str__(self):
    return self.__class__.__name__ + "(" + self.pathName + ")"

  def checkCall(self, *args, **kwArgs):
    assert type(*args) != types.StringType
    self.chDirToPath()
    if self.verbose:
      print "check_call args:", " ".join(*args)
    subprocess.check_call(*args, **kwArgs)

  def checkOutput(self, *args, **kwArgs):
    assert type(*args) != types.StringType
    self.chDirToPath()
    if self.verbose:
      print "check_output args:", " ".join(*args)
    result = subprocess.check_output(*args, **kwArgs)
    if self.verbose:
      print "check_output result:", result
    return result


class SvnWorkingCopy(SubProcessAtPath):
  def __init__(self, pathName):
    SubProcessAtPath.__init__(self, pathName, verbose=False)

  svnCmd = "svn"

  def ensureNoLocalModifications(self):
    localMods = self.checkOutput((self.svnCmd, "status"))
    if localMods:
      errorExit(self, "should not have local modifications:\n", localMods)

  def update(self):
    self.checkCall((self.svnCmd, "update"))

  def switch(self, repoBranchName):
    self.checkCall((self.svnCmd, "switch", ("^/" + repoBranchName)))

  def lastChangedRevision(self):
    infoXml = self.checkOutput((self.svnCmd, "info", "--xml"))
    infoHandler = SvnInfoHandler()
    sax.parseString(infoXml, infoHandler)
    return infoHandler.getLastChangeRevision()

  def getLogEntries(self, fromRevision, toRevision, maxNumLogEntries):
    revRange = self.revisionsRange(fromRevision, toRevision)
    logXml = self.checkOutput((self.svnCmd, "log", "-r", revRange, "--xml", "-l", str(maxNumLogEntries)))
    logHandler = SvnLogHandler()
    sax.parseString(logXml, logHandler)
    return logHandler.getLogEntries()

  def revisionsRange(self, fromRevision, toRevision):
    return str(fromRevision) + ":" + str(toRevision)

  def createPatchFile(self, fromRevision, toRevision, patchFileName):
    revRange = self.revisionsRange(fromRevision, toRevision)
    patchFile = open(patchFileName, 'w')
    try:
      print "Creating patch from", self.pathName, "between revisions", revRange
      self.checkCall((self.svnCmd, "diff", "-r", revRange,
                                    "--ignore-properties"), # git apply can fail on svn properties.
                     stdout=patchFile)
    finally:
      patchFile.close()
    print "Created patch file", patchFileName

  def patchedFileNames(self, patchFileName): # return a sequence of the patched file names
    if os.path.getsize(patchFileName) == 0: # changed only svn properties, no files changed.
      return []

    indexPrefix = "Index: "
    regExp = "^" + indexPrefix # at beginning of line
    patchedFileNamesLines = self.checkOutput(("grep", regExp, patchFileName)) # grep exits 1 whithout any match.
    indexPrefixLength = len(indexPrefix)
    return [line[indexPrefixLength:]
            for line in patchedFileNamesLines.split("\n")
            if len(line) > 0]


class GitRepository(SubProcessAtPath):
  def __init__(self, pathName):
    SubProcessAtPath.__init__(self, pathName, verbose=False)
    self.currentBranch = None

  gitCmd = "git"

  def checkOutBranch(self, branchName):
    self.checkCall((self.gitCmd, "checkout", branchName))
    self.currentBranch = branchName

  def getCurrentBranch(self):
    if self.currentBranch is None:
      gitStatusOut = self.checkOutput((self.gitCmd, "status"))
      if gitStatusOut.startswith("On branch "):
        self.currentBranch = gitStatusOut.split[2]
      else:
        errorExit(self, "not on a branch:", gitStatusOut)
    return self.currentBranch

  def workingDirectoryClean(self):
    gitStatusOut = self.checkOutput((self.gitCmd, "status"))
    expSubString = "nothing to commit, working directory clean"
    return gitStatusOut.find(expSubString) >= 0

  def listBranches(self, pattern):
    return self.checkOutput((self.gitCmd, "branch", "--list", pattern))

  def branchExists(self, branchName):
    listOut = self.listBranches(branchName) # CHECKME: using branchName as pattern may not always be ok.
    return len(listOut) > 0

  def deleteBranch(self, branchName):
    self.checkCall((self.gitCmd, "branch", "-D", branchName))
    if branchName == self.currentBranch:
      self.currentBranch = None

  def createBranch(self, branchName):
    self.checkCall((self.gitCmd, "branch", branchName))

  def fetch(self, upStream):
    self.checkCall((self.gitCmd, "fetch", upStream))

  def merge(self, branch, fromBranch):
    self.checkCall((self.gitCmd, "merge", branch, fromBranch))

  def getCommitMessage(self, commitRef):
    return self.checkOutput((self.gitCmd, "log", "--format=%B", "-n", "1", commitRef))

  def getCommitAuthorName(self, commitRef):
    return self.checkOutput((self.gitCmd, "log", "--format=%aN", "-n", "1", commitRef))

  def getCommitAuthorEmail(self, commitRef):
    return self.checkOutput((self.gitCmd, "log", "--format=%aE", "-n", "1", commitRef))

  def getLatestCommitForAuthor(self, svnAuthor):
    authorCommit = self.checkOutput(
                    " ".join((self.gitCmd,
                              "rev-list", "--all", "-i", ("--author=" + svnAuthor), # see git commit documentation on --author
                              "|",                               # pipe should have a buffer for at most a few commit ids.
                              "head", "-1")),
                    shell=True) # use shell pipe
    authorCommit = authorCommit.rstrip("\n")
    return authorCommit

  def getSvnRemoteAndRevision(self, gitSvnCommitRef):
    gitSvnCommitMessage = self.getCommitMessage(gitSvnCommitRef)
    words = gitSvnCommitMessage.split();
    svnIdMarker = "git-svn-id:"
    assert words.index(svnIdMarker) >= 0
    svnId = words[words.index(svnIdMarker) + 1]
    splitSvnId = svnId.split("@")
    svnRemote = splitSvnId[0]
    svnRevision = int(splitSvnId[1])
    return (svnRemote, svnRevision)

  def lastTempGitSvnRevision(self, branchName): # at a commit generated here on the temp branch.
    gitCommitMessage = self.getCommitMessage(branchName)
    parts = gitCommitMessage.split(":")
    lastPart = parts[-1].split()[0] # remove appended newlines
    try:
      return int(lastPart)
    except: # not generated here, ignore.
      print "Warning: svn revision range not found at end of commit message:\n", gitCommitMessage
      return None

  def applyPatch(self, patchFileName, stripDepth):
    self.checkCall((self.gitCmd, "apply",
                                  ("-p" + str(stripDepth)),
                                  "--whitespace=nowarn",
                                  patchFileName))

  def addAllToIndex(self):
    self.checkCall((self.gitCmd, "add", "-A"))

  def deleteForced(self, fileName):
    self.checkCall((self.gitCmd, "rm", "-f", fileName))

  def commit(self, message,
                  authorName, authorEmail, authorDate,
                  committerName, committerEmail, committerDate):
    author = ''.join((authorName, " <", authorEmail, ">"))
    os.environ["GIT_COMMITTER_NAME"] = committerName # no need to save/restore earlier environment state.
    os.environ["GIT_COMMITTER_EMAIL"] = committerEmail
    os.environ["GIT_COMMITTER_DATE"] = committerDate
    self.checkCall((self.gitCmd, "commit",
                                  "--allow-empty", # only svn poperties changed.
                                  ("--message=" + message),
                                  ("--author=" + author),
                                  ("--date=" + authorDate) ))

  def cleanDirsForced(self):
    self.checkCall((self.gitCmd, "clean", "-fd"))



def errorExit(*messageParts):
  raise RuntimeError(" ".join(map(str, messageParts)))


def allSuccessivePairs(lst):
  return [lst[i:i+2] for i in range(len(lst)-1)]


def maintainTempGitSvnBranch(branchName, tempGitBranchName,
                              svnWorkingCopyOfBranchPath, svnRepoBranchName,
                              gitRepoPath, gitUpstream,
                              patchFileName,
                              maxCommits=20, # generate at most this number of commits on tempGitBranchName, rerun to add more.
                              testMode=False):

  assert maxCommits >= 1

  gitRepo = GitRepository(gitRepoPath)
  gitRepo.checkOutBranch(branchName) # fails with git message when working directory is not clean

  svnWorkingCopy = SvnWorkingCopy(svnWorkingCopyOfBranchPath)
  svnWorkingCopy.ensureNoLocalModifications()
  svnWorkingCopy.switch(svnRepoBranchName) # switch to repo branch, update to latest revision

  lastSvnRevision = svnWorkingCopy.lastChangedRevision()
  # print svnWorkingCopy, "lastSvnRevision:", lastSvnRevision

  gitRepo.fetch(gitUpstream)
  if testMode:
    pass # leave branch where it is, as if the last commits from upstream did not arrive
  else:
    gitRepo.merge(branchName, gitUpstream + "/" + branchName)

  (svnRemote, lastSvnRevisionOnGitSvnBranch) = gitRepo.getSvnRemoteAndRevision(branchName)
  print "svnRemote:", svnRemote
  #print gitRepo, branchName, "lastSvnRevisionOnGitSvnBranch:", lastSvnRevisionOnGitSvnBranch

  # check whether tempGitBranchName exists:
  diffBaseRevision = lastSvnRevisionOnGitSvnBranch
  svnTempRevision = None
  doCommitOnExistingTempBranch = False

  if gitRepo.branchExists(tempGitBranchName):
    print tempGitBranchName, "exists"
    # update lastSvnRevisionOnGitSvnBranch from there.
    svnTempRevision = gitRepo.lastTempGitSvnRevision(tempGitBranchName)
    if svnTempRevision is None:
      print "Warning: no svn revision found on branch:", tempGitBranchName
    else:
      if svnTempRevision > lastSvnRevisionOnGitSvnBranch:
        diffBaseRevision = svnTempRevision
        doCommitOnExistingTempBranch = True
        gitRepo.checkOutBranch(tempGitBranchName)

  if lastSvnRevision == diffBaseRevision:
    print gitRepo, gitRepo.getCurrentBranch(), "up to date with", svnWorkingCopy, svnRepoBranchName
    return

  if lastSvnRevision < diffBaseRevision: # unlikely, do nothing
    print gitRepo, gitRepo.getCurrentBranch(), "later than", svnWorkingCopy, ", nothing to update."
    # CHECK: generate svn commits from the git commits?
    return

  print gitRepo, gitRepo.getCurrentBranch(), "earlier than", svnWorkingCopy

  if not gitRepo.workingDirectoryClean():
    errorExit(gitRepo, "on branch", gitRepo.getCurrentBranch(), "not clean")

  print gitRepo,"on branch", gitRepo.getCurrentBranch(), "and clean"

  if not doCommitOnExistingTempBranch: # restart temp branch from branch
    assert gitRepo.getCurrentBranch() == branchName
    if gitRepo.branchExists(tempGitBranchName): # tempGitBranchName exists, delete it first.
      print "Branch", tempGitBranchName, "exists, deleting"
      gitRepo.deleteBranch(tempGitBranchName)
      if gitRepo.branchExists(tempGitBranchName):
        errorExit("Could not delete branch", tempGitBranchName, "from", gitRepo)

    gitRepo.createBranch(tempGitBranchName)
    gitRepo.checkOutBranch(tempGitBranchName)
    print "Started branch", tempGitBranchName, "at", branchName

  assert gitRepo.getCurrentBranch() == tempGitBranchName

  patchStripDepth = 0 # patch generated at svn repo.

  maxNumLogEntries = maxCommits + 1
  svnLogEntries = svnWorkingCopy.getLogEntries(diffBaseRevision, lastSvnRevision, maxNumLogEntries)

  numCommits = 0

  for (logEntryFrom, logEntryTo) in allSuccessivePairs(svnLogEntries):
    # create patch file from svn between the revisions:
    svnWorkingCopy.createPatchFile(logEntryFrom.revision, logEntryTo.revision, patchFileName)

    patchedFileNames = svnWorkingCopy.patchedFileNames(patchFileName)

    if os.path.getsize(patchFileName) > 0:
      gitRepo.applyPatch(patchFileName, patchStripDepth)
      print "Applied patch", patchFileName
    else: # only svn properties changed, do git commit for commit info only.
      print "Empty patch", patchFileName

    gitRepo.addAllToIndex() # add all patch changes to the git index to be committed.

    # Applying the patch leaves files that have been actually deleted at zero size.
    # Therefore delete empty patched files from the git repo that do not exist in svn working copy:
    for patchedFileName in patchedFileNames:
      fileNameInGitRepo = os.path.join(gitRepo.getPathName(), patchedFileName)
      fileNameInSvnWorkingCopy = os.path.join(svnWorkingCopy.getPathName(), patchedFileName)

      if os.path.isdir(fileNameInGitRepo):
        # print "Directory:", fileNameInGitRepo
        continue

      if not os.path.isfile(fileNameInGitRepo):
        print "Possibly new binary file in svn, ignored here:", fileNameInGitRepo
        # FIXME: Take a new binary file out of the svn repository directly.
        continue

      fileSize = os.path.getsize(fileNameInGitRepo)
      if fileSize > 0:
        # print "Non empty file patched normally:", fileNameInGitRepo
        continue

      # fileNameInGitRepo exists and is empty
      if os.path.isfile(fileNameInSvnWorkingCopy):
        # FIXME: this only works correctly when the svn working copy is hecked out at the target revision.
        print "Left empty file:", fileNameInGitRepo
        continue

      gitRepo.deleteForced(fileNameInGitRepo) # force, the file is not up to date. This also stages the delete for commit.
      # print "Deleted empty file", fileNameInGitRepo # not needed, git rm is verbose enough

    # commit, put toRevision at end so it can be picked up later.
    revisionsRange = svnWorkingCopy.revisionsRange(logEntryFrom.revision, logEntryTo.revision)
    message = logEntryTo.msg + "\n\n" + svnRemote + " diff -r " + revisionsRange
    authorCommit = gitRepo.getLatestCommitForAuthor(logEntryTo.author)
    authorName = gitRepo.getCommitAuthorName(authorCommit)
    authorEmail = gitRepo.getCommitAuthorEmail(authorCommit)
    # print "Author name and email:", authorName, authorEmail
    gitRepo.commit(message,
                    authorName, authorEmail, logEntryTo.date,
                    authorName, authorEmail, logEntryTo.date) # author is also git committer, just like git-svn

    numCommits += 1

    # print "Commit  author:", logEntryTo.author
    # print "Commit    date:", logEntryTo.date
    print "Commit message:", logEntryTo.msg

    gitRepo.cleanDirsForced() # delete untracked directories and files

    if not gitRepo.workingDirectoryClean():
      errorExit(gitRepo, "on branch", gitRepo.getCurrentBranch(), "not clean, numCommits:", numCommits)

  print "Added", numCommits, "commit(s) to branch", tempGitBranchName


if __name__ == "__main__":

  testMode = False # when true, leave branch where it is, as if the last commits from upstream did not arrive
  defaultMaxCommits = 20
  maxCommits = defaultMaxCommits

  import sys
  argv = sys.argv[1:]
  while argv:
    if argv[0] == "test":
      testMode = True
    else:
      try:
        maxCommits = int(argv[0])
        assert maxCommits >= 1
      except:
        errorExit("Argument(s) should be test and/or a maximum number of commits, defaults are false and " + defaultMaxCommits)
    argv = argv[1:]

  repo = "lucene-solr"
  branchName = "trunk"
  tempGitBranchName = branchName + ".svn"

  home = os.path.expanduser("~")

  svnWorkingCopyOfBranchPath = os.path.join(home, "svnwork", repo, branchName)
  svnRepoBranchName = "lucene/dev/" + branchName # for svn switch to

  gitRepo = os.path.join(home, "gitrepos", repo)
  gitUpstream = "upstream"

  patchFileName = os.path.join(home, "patches", tempGitBranchName)

  maintainTempGitSvnBranch(branchName, tempGitBranchName,
                            svnWorkingCopyOfBranchPath, svnRepoBranchName,
                            gitRepo, gitUpstream,
                            patchFileName,
                            maxCommits=maxCommits,
                            testMode=testMode)