lucene/dev-tools/scripts/releaseWizard.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is the Release Manager's best friend, ensuring all details of a release are handled correctly.
# It will walk you through the steps of the release process, asking for decisions or input along the way.
# CAUTION: You still need to use your head! Please read the HELP section in the main menu.
#
# Requirements:
#   Install requirements with this command:
#   pip3 install -r requirements.txt
#
# Usage:
#   releaseWizard.py [-h] [--dry-run] [--root PATH]
#
#   optional arguments:
#   -h, --help   show this help message and exit
#   --dry-run    Do not execute any commands, but echo them instead. Display
#   extra debug info
#   --root PATH  Specify different root folder than ~/.lucene-releases

import argparse
import copy
import fcntl
import json
import os
import platform
import re
import shlex
import shutil
import subprocess
import sys
import textwrap
import time
import urllib
from collections import OrderedDict
from datetime import datetime
from datetime import timedelta

try:
    import holidays
    import yaml
    from ics import Calendar, Event
    from jinja2 import Environment
except:
    print("You lack some of the module dependencies to run this script.")
    print("Please run 'pip3 install -r requirements.txt' and try again.")
    sys.exit(1)

import scriptutil
from consolemenu import ConsoleMenu
from consolemenu.items import FunctionItem, SubmenuItem, ExitItem
from consolemenu.screen import Screen
from scriptutil import BranchType, Version, download, run

# Solr-to-Java version mapping
java_versions = {6: 8, 7: 8, 8: 8, 9: 11}
editor = None

# Edit this to add other global jinja2 variables or filters
def expand_jinja(text, vars=None):
    global_vars = OrderedDict({
        'script_version': state.script_version,
        'release_version': state.release_version,
        'release_version_underscore': state.release_version.replace('.', '_'),
        'release_date': state.get_release_date(),
        'ivy2_folder': os.path.expanduser("~/.ivy2/"),
        'config_path': state.config_path,
        'rc_number': state.rc_number,
        'script_branch': state.script_branch,
        'release_folder': state.get_release_folder(),
        'git_checkout_folder': state.get_git_checkout_folder(),
        'git_website_folder': state.get_website_git_folder(),
        'dist_url_base': 'https://dist.apache.org/repos/dist/dev/lucene',
        'm2_repository_url': 'https://repository.apache.org/service/local/staging/deploy/maven2',
        'dist_file_path': state.get_dist_folder(),
        'rc_folder': state.get_rc_folder(),
        'base_branch': state.get_base_branch_name(),
        'release_branch': state.release_branch,
        'stable_branch': state.get_stable_branch_name(),
        'minor_branch': state.get_minor_branch_name(),
        'release_type': state.release_type,
        'is_feature_release': state.release_type in ['minor', 'major'],
        'release_version_major': state.release_version_major,
        'release_version_minor': state.release_version_minor,
        'release_version_bugfix': state.release_version_bugfix,
        'release_version_refguide': state.get_refguide_release() ,
        'state': state,
        'gpg_key' : state.get_gpg_key(),
        'gradle_cmd' : 'gradlew.bat' if is_windows() else './gradlew',
        'epoch': unix_time_millis(datetime.utcnow()),
        'get_next_version': state.get_next_version(),
        'current_git_rev': state.get_current_git_rev(),
        'keys_downloaded': keys_downloaded(),
        'editor': get_editor(),
        'rename_cmd': 'ren' if is_windows() else 'mv',
        'vote_close_72h': vote_close_72h_date().strftime("%Y-%m-%d %H:00 UTC"),
        'vote_close_72h_epoch': unix_time_millis(vote_close_72h_date()),
        'vote_close_72h_holidays': vote_close_72h_holidays(),
        'lucene_news_file': lucene_news_file,
        'solr_news_file': solr_news_file,
        'load_lines': load_lines,
        'set_java_home': set_java_home,
        'latest_version': state.get_latest_version(),
        'latest_lts_version': state.get_latest_lts_version(),
        'master_version': state.get_master_version(),
        'mirrored_versions': state.get_mirrored_versions(),
        'mirrored_versions_to_delete': state.get_mirrored_versions_to_delete(),
        'home': os.path.expanduser("~")
    })
    global_vars.update(state.get_todo_states())
    if vars:
        global_vars.update(vars)

    filled = replace_templates(text)

    try:
        env = Environment(lstrip_blocks=True, keep_trailing_newline=False, trim_blocks=True)
        env.filters['path_join'] = lambda paths: os.path.join(*paths)
        env.filters['expanduser'] = lambda path: os.path.expanduser(path)
        env.filters['formatdate'] = lambda date: (datetime.strftime(date, "%-d %B %Y") if date else "<date>" )
        template = env.from_string(str(filled), globals=global_vars)
        filled = template.render()
    except Exception as e:
        print("Exception while rendering jinja template %s: %s" % (str(filled)[:10], e))
    return filled


def replace_templates(text):
    tpl_lines = []
    for line in text.splitlines():
        if line.startswith("(( template="):
            match = re.search(r"^\(\( template=(.+?) \)\)", line)
            name = match.group(1)
            tpl_lines.append(replace_templates(templates[name].strip()))
        else:
            tpl_lines.append(line)
    return "\n".join(tpl_lines)

def getScriptVersion():
    return scriptutil.find_current_version()


def get_editor():
    global editor
    if editor is None:
      if 'EDITOR' in os.environ:
          if os.environ['EDITOR'] in ['vi', 'vim', 'nano', 'pico', 'emacs']:
              print("WARNING: You have EDITOR set to %s, which will not work when launched from this tool. Please use an editor that launches a separate window/process" % os.environ['EDITOR'])
          editor = os.environ['EDITOR']
      elif is_windows():
          editor = 'notepad.exe'
      elif is_mac():
          editor = 'open -a TextEdit'
      else:
          sys.exit("On Linux you have to set EDITOR variable to a command that will start an editor in its own window")
    return editor


def check_prerequisites(todo=None):
    if sys.version_info < (3, 4):
        sys.exit("Script requires Python v3.4 or later")
    try:
        gpg_ver = run("gpg --version").splitlines()[0]
    except:
        sys.exit("You will need gpg installed")
    if not 'GPG_TTY' in os.environ:
        print("WARNING: GPG_TTY environment variable is not set, GPG signing may not work correctly (try 'export GPG_TTY=$(tty)'")
    if not 'JAVA11_HOME' in os.environ:
        sys.exit("Please set environment variables JAVA11_HOME")
    try:
        asciidoc_ver = run("asciidoctor -V").splitlines()[0]
    except:
        asciidoc_ver = ""
        print("WARNING: In order to export asciidoc version to HTML, you will need asciidoctor installed")
    try:
        git_ver = run("git --version").splitlines()[0]
    except:
        sys.exit("You will need git installed")
    if not 'EDITOR' in os.environ:
        print("WARNING: Environment variable $EDITOR not set, using %s" % get_editor())

    if todo:
        print("%s\n%s\n%s\n" % (gpg_ver, asciidoc_ver, git_ver))
    return True


epoch = datetime.utcfromtimestamp(0)


def unix_time_millis(dt):
    return int((dt - epoch).total_seconds() * 1000.0)


def bootstrap_todos(todo_list):
    # Establish links from commands to to_do for finding todo vars
    for tg in todo_list:
        if dry_run:
            print("Group %s" % tg.id)
        for td in tg.get_todos():
            if dry_run:
                print("  Todo %s" % td.id)
            cmds = td.commands
            if cmds:
                if dry_run:
                    print("  Commands")
                cmds.todo_id = td.id
                for cmd in cmds.commands:
                    if dry_run:
                        print("    Command %s" % cmd.cmd)
                    cmd.todo_id = td.id

    print("Loaded TODO definitions from releaseWizard.yaml")
    return todo_list


def maybe_remove_rc_from_svn():
    todo = state.get_todo_by_id('import_svn')
    if todo and todo.is_done():
        print("import_svn done")
        Commands(state.get_git_checkout_folder(),
                 """Looks like you uploaded artifacts for {{ build_rc.git_rev | default("<git_rev>", True) }} to svn which needs to be removed.""",
                 [Command(
                 """svn -m "Remove cancelled Lucene/Solr {{ release_version }} RC{{ rc_number }}" rm {{ dist_url }}""",
                 logfile="svn_rm.log",
                 tee=True,
                 vars={
                     'dist_folder': """lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}""",
                     'dist_url': "{{ dist_url_base }}/{{ dist_folder }}"
                 }
             )],
                 enable_execute=True, confirm_each_command=False).run()


# To be able to hide fields when dumping Yaml
class SecretYamlObject(yaml.YAMLObject):
    hidden_fields = []
    @classmethod
    def to_yaml(cls,dumper,data):
        print("Dumping object %s" % type(data))

        new_data = copy.deepcopy(data)
        for item in cls.hidden_fields:
            if item in new_data.__dict__:
                del new_data.__dict__[item]
        for item in data.__dict__:
            if item in new_data.__dict__ and new_data.__dict__[item] is None:
                del new_data.__dict__[item]
        return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls,
                                            flow_style=cls.yaml_flow_style)


def str_presenter(dumper, data):
    if len(data.split('\n')) > 1:  # check for multiline string
        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
    return dumper.represent_scalar('tag:yaml.org,2002:str', data)


class ReleaseState:
    def __init__(self, config_path, release_version, script_version):
        self.script_version = script_version
        self.config_path = config_path
        self.todo_groups = None
        self.todos = None
        self.latest_version = None
        self.previous_rcs = {}
        self.rc_number = 1
        self.start_date = unix_time_millis(datetime.utcnow())
        self.script_branch = run("git rev-parse --abbrev-ref HEAD").strip()
        self.mirrored_versions = None
        try:
            self.script_branch_type = scriptutil.find_branch_type()
        except:
            print("WARNING: This script shold (ideally) run from the release branch, not a feature branch (%s)" % self.script_branch)
            self.script_branch_type = 'feature'
        self.set_release_version(release_version)

    def set_release_version(self, version):
        self.validate_release_version(self.script_branch_type, self.script_branch, version)
        self.release_version = version
        v = Version.parse(version)
        self.release_version_major = v.major
        self.release_version_minor = v.minor
        self.release_version_bugfix = v.bugfix
        self.release_branch = "branch_%s_%s" % (v.major, v.minor)
        if v.is_major_release():
            self.release_type = 'major'
        elif v.is_minor_release():
            self.release_type = 'minor'
        else:
            self.release_type = 'bugfix'

    def is_released(self):
        return self.get_todo_by_id('announce_lucene').is_done()

    def get_gpg_key(self):
        gpg_task = self.get_todo_by_id('gpg')
        if gpg_task.is_done():
            return gpg_task.get_state()['gpg_key']
        else:
            return None

    def get_release_date(self):
        publish_task = self.get_todo_by_id('publish_maven')
        if publish_task.is_done():
            return unix_to_datetime(publish_task.get_state()['done_date'])
        else:
            return None

    def get_release_date_iso(self):
        release_date = self.get_release_date()
        if release_date is None:
            return "yyyy-mm-dd"
        else:
            return release_date.isoformat()[:10]

    def get_latest_version(self):
        if self.latest_version is None:
            versions = self.get_mirrored_versions()
            latest = versions[0]
            for ver in versions:
                if Version.parse(ver).gt(Version.parse(latest)):
                    latest = ver
            self.latest_version = latest
            self.save()
        return state.latest_version

    def get_mirrored_versions(self):
        if state.mirrored_versions is None:
            releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")
            releases = json.loads(releases_str)['lucene']
            state.mirrored_versions = [ r for r in list(map(lambda y: y[7:], filter(lambda x: x.startswith('lucene-'), list(releases.keys())))) ]
        return state.mirrored_versions

    def get_mirrored_versions_to_delete(self):
        versions = self.get_mirrored_versions()
        to_keep = versions
        if state.release_type == 'major':
          to_keep = [self.release_version, self.get_latest_version()]
        if state.release_type == 'minor':
          to_keep = [self.release_version, self.get_latest_lts_version()]
        if state.release_type == 'bugfix':
          if Version.parse(state.release_version).major == Version.parse(state.get_latest_version()).major:
            to_keep = [self.release_version, self.get_latest_lts_version()]
          elif Version.parse(state.release_version).major == Version.parse(state.get_latest_lts_version()).major:
            to_keep = [self.get_latest_version(), self.release_version]
          else:
            raise Exception("Release version %s must have same major version as current minor or lts release")
        return [ver for ver in versions if ver not in to_keep]

    def get_master_version(self):
        v = Version.parse(self.get_latest_version())
        return "%s.%s.%s" % (v.major + 1, 0, 0)

    def get_latest_lts_version(self):
        versions = self.get_mirrored_versions()
        latest = self.get_latest_version()
        lts_prefix = "%s." % (Version.parse(latest).major - 1)
        lts_versions = list(filter(lambda x: x.startswith(lts_prefix), versions))
        latest_lts = lts_versions[0]
        for ver in lts_versions:
            if Version.parse(ver).gt(Version.parse(latest_lts)):
                latest_lts = ver
        return latest_lts

    def validate_release_version(self, branch_type, branch, release_version):
        ver = Version.parse(release_version)
        # print("release_version=%s, ver=%s" % (release_version, ver))
        if branch_type == BranchType.release:
            if not branch.startswith('branch_'):
                sys.exit("Incompatible branch and branch_type")
            if not ver.is_bugfix_release():
                sys.exit("You can only release bugfix releases from an existing release branch")
        elif branch_type == BranchType.stable:
            if not branch.startswith('branch_') and branch.endswith('x'):
                sys.exit("Incompatible branch and branch_type")
            if not ver.is_minor_release():
                sys.exit("You can only release minor releases from an existing stable branch")
        elif branch_type == BranchType.unstable:
            if not branch == 'master':
                sys.exit("Incompatible branch and branch_type")
            if not ver.is_major_release():
                sys.exit("You can only release a new major version from master branch")
        if not getScriptVersion() == release_version:
            print("WARNING: Expected release version %s when on branch %s, but got %s" % (
                getScriptVersion(), branch, release_version))

    def get_base_branch_name(self):
        v = Version.parse(self.release_version)
        if v.is_major_release():
            return 'master'
        elif v.is_minor_release():
            return self.get_stable_branch_name()
        elif v.major == Version.parse(self.get_latest_version()).major:
            return self.get_minor_branch_name()
        else:
            return self.release_branch

    def clear_rc(self):
        if ask_yes_no("Are you sure? This will clear and restart RC%s" % self.rc_number):
            maybe_remove_rc_from_svn()
            dict = {}
            for g in list(filter(lambda x: x.in_rc_loop(), self.todo_groups)):
                for t in g.get_todos():
                    t.clear()
            print("Cleared RC TODO state")
            try:
                shutil.rmtree(self.get_rc_folder())
                print("Cleared folder %s" % self.get_rc_folder())
            except Exception as e:
                print("WARN: Failed to clear %s, please do it manually with higher privileges" % self.get_rc_folder())
            self.save()

    def new_rc(self):
        if ask_yes_no("Are you sure? This will abort current RC"):
            maybe_remove_rc_from_svn()
            dict = {}
            for g in list(filter(lambda x: x.in_rc_loop(), self.todo_groups)):
                for t in g.get_todos():
                    if t.applies(self.release_type):
                        dict[t.id] = copy.deepcopy(t.state)
                        t.clear()
            self.previous_rcs["RC%d" % self.rc_number] = dict
            self.rc_number += 1
            self.save()

    def to_dict(self):
        tmp_todos = {}
        for todo_id in self.todos:
            t = self.todos[todo_id]
            tmp_todos[todo_id] = copy.deepcopy(t.state)
        dict = {
            'script_version': self.script_version,
            'release_version': self.release_version,
            'start_date': self.start_date,
            'rc_number': self.rc_number,
            'script_branch': self.script_branch,
            'todos': tmp_todos,
            'previous_rcs': self.previous_rcs
        }
        if self.latest_version:
            dict['latest_version'] = self.latest_version
        return dict

    def restore_from_dict(self, dict):
        self.script_version = dict['script_version']
        assert dict['release_version'] == self.release_version
        if 'start_date' in dict:
            self.start_date = dict['start_date']
        if 'latest_version' in dict:
            self.latest_version = dict['latest_version']
        else:
            self.latest_version = None
        self.rc_number = dict['rc_number']
        self.script_branch = dict['script_branch']
        self.previous_rcs = copy.deepcopy(dict['previous_rcs'])
        for todo_id in dict['todos']:
            if todo_id in self.todos:
                t = self.todos[todo_id]
                for k in dict['todos'][todo_id]:
                    t.state[k] = dict['todos'][todo_id][k]
            else:
                print("Warning: Could not restore state for %s, Todo definition not found" % todo_id)

    def load(self):
        if os.path.exists(os.path.join(self.config_path, self.release_version, 'state.yaml')):
            state_file = os.path.join(self.config_path, self.release_version, 'state.yaml')
            with open(state_file, 'r') as fp:
                try:
                    dict = yaml.load(fp, Loader=yaml.Loader)
                    self.restore_from_dict(dict)
                    print("Loaded state from %s" % state_file)
                except Exception as e:
                    print("Failed to load state from %s: %s" % (state_file, e))

    def save(self):
        print("Saving")
        if not os.path.exists(os.path.join(self.config_path, self.release_version)):
            print("Creating folder %s" % os.path.join(self.config_path, self.release_version))
            os.makedirs(os.path.join(self.config_path, self.release_version))

        with open(os.path.join(self.config_path, self.release_version, 'state.yaml'), 'w') as fp:
            yaml.dump(self.to_dict(), fp, sort_keys=False, default_flow_style=False)

    def clear(self):
        self.previous_rcs = {}
        self.rc_number = 1
        for t_id in self.todos:
            t = self.todos[t_id]
            t.state = {}
        self.save()

    def get_rc_number(self):
        return self.rc_number

    def get_current_git_rev(self):
        try:
            return run("git rev-parse HEAD", cwd=self.get_git_checkout_folder()).strip()
        except:
            return "<git-rev>"

    def get_group_by_id(self, id):
        lst = list(filter(lambda x: x.id == id, self.todo_groups))
        if len(lst) == 1:
            return lst[0]
        else:
            return None

    def get_todo_by_id(self, id):
        lst = list(filter(lambda x: x.id == id, self.todos.values()))
        if len(lst) == 1:
            return lst[0]
        else:
            return None

    def get_todo_state_by_id(self, id):
        lst = list(filter(lambda x: x.id == id, self.todos.values()))
        if len(lst) == 1:
            return lst[0].state
        else:
            return {}

    def get_release_folder(self):
        folder = os.path.join(self.config_path, self.release_version)
        if not os.path.exists(folder):
            print("Creating folder %s" % folder)
            os.makedirs(folder)
        return folder

    def get_rc_folder(self):
        folder = os.path.join(self.get_release_folder(), "RC%d" % self.rc_number)
        if not os.path.exists(folder):
            print("Creating folder %s" % folder)
            os.makedirs(folder)
        return folder

    def get_dist_folder(self):
        folder = os.path.join(self.get_rc_folder(), "dist")
        return folder

    def get_git_checkout_folder(self):
        folder = os.path.join(self.get_release_folder(), "lucene-solr")
        return folder

    def get_website_git_folder(self):
        folder = os.path.join(self.get_release_folder(), "lucene-site")
        return folder

    def get_minor_branch_name(self):
        latest = state.get_latest_version()
        if latest is not None:
          v = Version.parse(latest)
          return "branch_%s_%s" % (v.major, v.minor)
        else:
            raise Exception("Cannot find latest version")

    def get_stable_branch_name(self):
        if self.release_type == 'major':
            v = Version.parse(self.get_master_version())
        else:
            v = Version.parse(self.get_latest_version())
        return "branch_%sx" % v.major

    def get_next_version(self):
        if self.release_type == 'major':
            return "%s.0.0" % (self.release_version_major + 1)
        if self.release_type == 'minor':
            return "%s.%s.0" % (self.release_version_major, self.release_version_minor + 1)
        if self.release_type == 'bugfix':
            return "%s.%s.%s" % (self.release_version_major, self.release_version_minor, self.release_version_bugfix + 1)

    def get_refguide_release(self):
        return "%s_%s" % (self.release_version_major, self.release_version_minor)

    def get_java_home(self):
        return self.get_java_home_for_version(self.release_version)

    def get_java_home_for_version(self, version):
        v = Version.parse(version)
        java_ver = java_versions[v.major]
        java_home_var = "JAVA%s_HOME" % java_ver
        if java_home_var in os.environ:
            return os.environ.get(java_home_var)
        else:
            raise Exception("Script needs environment variable %s" % java_home_var )

    def get_java_cmd_for_version(self, version):
        return os.path.join(self.get_java_home_for_version(version), "bin", "java")

    def get_java_cmd(self):
        return os.path.join(self.get_java_home(), "bin", "java")

    def get_todo_states(self):
        states = {}
        if self.todos:
            for todo_id in self.todos:
                t = self.todos[todo_id]
                states[todo_id] = copy.deepcopy(t.state)
        return states

    def init_todos(self, groups):
        self.todo_groups = groups
        self.todos = {}
        for g in self.todo_groups:
            for t in g.get_todos():
                self.todos[t.id] = t


class TodoGroup(SecretYamlObject):
    yaml_tag = u'!TodoGroup'
    hidden_fields = []
    def __init__(self, id, title, description, todos, is_in_rc_loop=None, depends=None):
        self.id = id
        self.title = title
        self.description = description
        self.depends = depends
        self.is_in_rc_loop = is_in_rc_loop
        self.todos = todos

    @classmethod
    def from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node, deep = True)
        return TodoGroup(**fields)

    def num_done(self):
        return sum(1 for x in self.todos if x.is_done() > 0)

    def num_applies(self):
        count = sum(1 for x in self.todos if x.applies(state.release_type))
        # print("num_applies=%s" % count)
        return count

    def is_done(self):
        # print("Done=%s, applies=%s" % (self.num_done(), self.num_applies()))
        return self.num_done() >= self.num_applies()

    def get_title(self):
        # print("get_title: %s" % self.is_done())
        prefix = ""
        if self.is_done():
            prefix = "✓ "
        return "%s%s (%d/%d)" % (prefix, self.title, self.num_done(), self.num_applies())

    def get_submenu(self):
        menu = UpdatableConsoleMenu(title=self.title, subtitle=self.get_subtitle, prologue_text=self.get_description(),
                           screen=MyScreen())
        menu.exit_item = CustomExitItem("Return")
        for todo in self.get_todos():
            if todo.applies(state.release_type):
                menu.append_item(todo.get_menu_item())
        return menu

    def get_menu_item(self):
        item = UpdatableSubmenuItem(self.get_title, self.get_submenu())
        return item

    def get_todos(self):
        return self.todos

    def in_rc_loop(self):
        return self.is_in_rc_loop is True

    def get_description(self):
        desc = self.description
        if desc:
            return expand_jinja(desc)
        else:
            return None

    def get_subtitle(self):
        if self.depends:
            ret_str = ""
            for dep in ensure_list(self.depends):
                g = state.get_group_by_id(dep)
                if not g:
                    g = state.get_todo_by_id(dep)
                if g and not g.is_done():
                    ret_str += "NOTE: Please first complete '%s'\n" % g.title
                    return ret_str.strip()
        return None


class Todo(SecretYamlObject):
    yaml_tag = u'!Todo'
    hidden_fields = ['state']
    def __init__(self, id, title, description=None, post_description=None, done=None, types=None, links=None,
                 commands=None, user_input=None, depends=None, vars=None, asciidoc=None, persist_vars=None,
                 function=None):
        self.id = id
        self.title = title
        self.description = description
        self.asciidoc = asciidoc
        self.types = types
        self.depends = depends
        self.vars = vars
        self.persist_vars = persist_vars
        self.function = function
        self.user_input = user_input
        self.commands = commands
        self.post_description = post_description
        self.links = links
        self.state = {}

        self.set_done(done)
        if self.types:
            self.types = ensure_list(self.types)
            for t in self.types:
                if not t in ['minor', 'major', 'bugfix']:
                    sys.exit("Wrong Todo config for '%s'. Type needs to be either 'minor', 'major' or 'bugfix'" % self.id)
        if commands:
            self.commands.todo_id = self.id
            for c in commands.commands:
                c.todo_id = self.id

    @classmethod
    def from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node, deep = True)
        return Todo(**fields)

    def get_vars(self):
        myvars = {}
        if self.vars:
            for k in self.vars:
                val = self.vars[k]
                if callable(val):
                    myvars[k] = expand_jinja(val(), vars=myvars)
                else:
                    myvars[k] = expand_jinja(val, vars=myvars)
        return myvars

    def set_done(self, is_done):
        if is_done:
            self.state['done_date'] = unix_time_millis(datetime.utcnow())
            if self.persist_vars:
                for k in self.persist_vars:
                    self.state[k] = self.get_vars()[k]
        else:
            self.state.clear()
        self.state['done'] = is_done

    def applies(self, type):
        if self.types:
            return type in self.types
        return True

    def is_done(self):
        return 'done' in self.state and self.state['done'] is True

    def get_title(self):
        prefix = ""
        if self.is_done():
            prefix = "✓ "
        return expand_jinja("%s%s" % (prefix, self.title), self.get_vars_and_state())

    def display_and_confirm(self):
        try:
            if self.depends:
                ret_str = ""
                for dep in ensure_list(self.depends):
                    g = state.get_group_by_id(dep)
                    if not g:
                        g = state.get_todo_by_id(dep)
                    if not g.is_done():
                        print("This step depends on '%s'. Please complete that first\n" % g.title)
                        return
            desc = self.get_description()
            if desc:
                print("%s" % desc)
            try:
                if self.function and not self.is_done():
                    if not eval(self.function)(self):
                        return
            except Exception as e:
                print("Function call to %s for todo %s failed: %s" % (self.function, self.id, e))
                raise e
            if self.user_input and not self.is_done():
                ui_list = ensure_list(self.user_input)
                for ui in ui_list:
                    ui.run(self.state)
                print()
            if self.links:
                print("\nLinks:\n")
                for link in self.links:
                    print("- %s" % expand_jinja(link, self.get_vars_and_state()))
                print()
            cmds = self.get_commands()
            if cmds:
                if not self.is_done():
                    if not cmds.logs_prefix:
                        cmds.logs_prefix = self.id
                    cmds.run()
                else:
                    print("This step is already completed. You have to first set it to 'not completed' in order to execute commands again.")
                print()
            if self.post_description:
                print("%s" % self.get_post_description())
            todostate = self.get_state()
            if self.is_done() and len(todostate) > 2:
                print("Variables registered\n")
                for k in todostate:
                    if k == 'done' or k == 'done_date':
                        continue
                    print("* %s = %s" % (k, todostate[k]))
                print()
            completed = ask_yes_no("Mark task '%s' as completed?" % self.get_title())
            self.set_done(completed)
            state.save()
        except Exception as e:
            print("ERROR while executing todo %s (%s)" % (self.get_title(), e))

    def get_menu_item(self):
        return UpdatableFunctionItem(self.get_title, self.display_and_confirm)

    def clone(self):
        clone = Todo(self.id, self.title, description=self.description)
        clone.state = copy.deepcopy(self.state)
        return clone

    def clear(self):
        self.state.clear()
        self.set_done(False)

    def get_state(self):
        return self.state

    def get_description(self):
        desc = self.description
        if desc:
            return expand_jinja(desc, vars=self.get_vars_and_state())
        else:
            return None

    def get_post_description(self):
        if self.post_description:
            return expand_jinja(self.post_description, vars=self.get_vars_and_state())
        else:
            return None

    def get_commands(self):
        cmds = self.commands
        return cmds

    def get_asciidoc(self):
        if self.asciidoc:
            return expand_jinja(self.asciidoc, vars=self.get_vars_and_state())
        else:
            return None

    def get_vars_and_state(self):
        d = self.get_vars().copy()
        d.update(self.get_state())
        return d


def get_release_version():
    v = str(input("Which version are you releasing? (x.y.z) "))
    try:
        version = Version.parse(v)
    except:
        print("Not a valid version %s" % v)
        return get_release_version()

    return str(version)


def get_subtitle():
    applying_groups = list(filter(lambda x: x.num_applies() > 0, state.todo_groups))
    done_groups = sum(1 for x in applying_groups if x.is_done())
    return "Please complete the below checklist (Complete: %s/%s)" % (done_groups, len(applying_groups))


def get_todo_menuitem_title():
    return "Go to checklist (RC%d)" % (state.rc_number)


def get_releasing_text():
    return "Releasing Lucene/Solr %s RC%d" % (state.release_version, state.rc_number)


def get_start_new_rc_menu_title():
    return "Abort RC%d and start a new RC%d" % (state.rc_number, state.rc_number + 1)


def start_new_rc():
    state.new_rc()
    print("Started RC%d" % state.rc_number)


def reset_state():
    global state
    if ask_yes_no("Are you sure? This will erase all current progress"):
        maybe_remove_rc_from_svn()
        shutil.rmtree(os.path.join(state.config_path, state.release_version))
        state.clear()


def template(name, vars=None):
    return expand_jinja(templates[name], vars=vars)


def help():
    print(template('help'))
    pause()


def ensure_list(o):
    if o is None:
        return []
    if not isinstance(o, list):
        return [o]
    else:
        return o


def open_file(filename):
    print("Opening file %s" % filename)
    if platform.system().startswith("Win"):
        run("start %s" % filename)
    else:
        run("open %s" % filename)


def expand_multiline(cmd_txt, indent=0):
    return re.sub(r'  +', " %s\n    %s" % (Commands.cmd_continuation_char, " "*indent), cmd_txt)


def unix_to_datetime(unix_stamp):
    return datetime.utcfromtimestamp(unix_stamp / 1000)


def generate_asciidoc():
    base_filename = os.path.join(state.get_release_folder(),
                                 "lucene_solr_release_%s"
                                 % (state.release_version.replace("\.", "_")))

    filename_adoc = "%s.adoc" % base_filename
    filename_html = "%s.html" % base_filename
    fh = open(filename_adoc, "w")

    fh.write("= Lucene/Solr Release %s\n\n" % state.release_version)
    fh.write("(_Generated by releaseWizard.py v%s at %s_)\n\n"
             % (getScriptVersion(), datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")))
    fh.write(":numbered:\n\n")
    fh.write("%s\n\n" % template('help'))
    for group in state.todo_groups:
        if group.num_applies() == 0:
            continue
        fh.write("== %s\n\n" % group.get_title())
        fh.write("%s\n\n" % group.get_description())
        for todo in group.get_todos():
            if not todo.applies(state.release_type):
                continue
            fh.write("=== %s\n\n" % todo.get_title())
            if todo.is_done():
                fh.write("_Completed %s_\n\n" % unix_to_datetime(todo.state['done_date']).strftime(
                    "%Y-%m-%d %H:%M UTC"))
            if todo.get_asciidoc():
                fh.write("%s\n\n" % todo.get_asciidoc())
            else:
                desc = todo.get_description()
                if desc:
                    fh.write("%s\n\n" % desc)
            state_copy = copy.deepcopy(todo.state)
            state_copy.pop('done', None)
            state_copy.pop('done_date', None)
            if len(state_copy) > 0 or todo.user_input is not None:
                fh.write(".Variables collected in this step\n")
                fh.write("|===\n")
                fh.write("|Variable |Value\n")
                mykeys = set()
                for e in ensure_list(todo.user_input):
                    mykeys.add(e.name)
                for e in state_copy.keys():
                    mykeys.add(e)
                for key in mykeys:
                    val = "(not set)"
                    if key in state_copy:
                        val = state_copy[key]
                    fh.write("\n|%s\n|%s\n" % (key, val))
                fh.write("|===\n\n")
            cmds = todo.get_commands()
            if cmds:
                if cmds.commands_text:
                    fh.write("%s\n\n" % cmds.get_commands_text())
                fh.write("[source,sh]\n----\n")
                if cmds.env:
                    for key in cmds.env:
                        val = cmds.env[key]
                        if is_windows():
                            fh.write("SET %s=%s\n" % (key, val))
                        else:
                            fh.write("export %s=%s\n" % (key, val))
                fh.write(abbreviate_homedir("cd %s\n" % cmds.get_root_folder()))
                cmds2 = ensure_list(cmds.commands)
                for c in cmds2:
                    for line in c.display_cmd():
                        fh.write("%s\n" % line)
                fh.write("----\n\n")
            if todo.post_description and not todo.get_asciidoc():
                fh.write("\n%s\n\n" % todo.get_post_description())
            if todo.links:
                fh.write("Links:\n\n")
                for l in todo.links:
                    fh.write("* %s\n" % expand_jinja(l))
                fh.write("\n")

    fh.close()
    print("Wrote file %s" % os.path.join(state.get_release_folder(), filename_adoc))
    print("Running command 'asciidoctor %s'" % filename_adoc)
    run_follow("asciidoctor %s" % filename_adoc)
    if os.path.exists(filename_html):
        open_file(filename_html)
    else:
        print("Failed generating HTML version, please install asciidoctor")
    pause()


def load_rc():
    lucenerc = os.path.expanduser("~/.lucenerc")
    try:
        with open(lucenerc, 'r') as fp:
            return json.load(fp)
    except:
        return None


def store_rc(release_root, release_version=None):
    lucenerc = os.path.expanduser("~/.lucenerc")
    dict = {}
    dict['root'] = release_root
    if release_version:
        dict['release_version'] = release_version
    with open(lucenerc, "w") as fp:
        json.dump(dict, fp, indent=2)


def release_other_version():
    if not state.is_released():
        maybe_remove_rc_from_svn()
    store_rc(state.config_path, None)
    print("Please restart the wizard")
    sys.exit(0)

def file_to_string(filename):
    with open(filename, encoding='utf8') as f:
        return f.read().strip()

def download_keys():
    download('KEYS', "https://archive.apache.org/dist/lucene/KEYS", state.config_path)

def keys_downloaded():
    return os.path.exists(os.path.join(state.config_path, "KEYS"))


def dump_yaml():
    file = open(os.path.join(script_path, "releaseWizard.yaml"), "w")
    yaml.add_representer(str, str_presenter)
    yaml.Dumper.ignore_aliases = lambda *args : True
    dump_obj = {'templates': templates,
                'groups': state.todo_groups}
    yaml.dump(dump_obj, width=180, stream=file, sort_keys=False, default_flow_style=False)


def parse_config():
    description = 'Script to guide a RM through the whole release process'
    parser = argparse.ArgumentParser(description=description, epilog="Go push that release!",
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--dry-run', dest='dry', action='store_true', default=False,
                        help='Do not execute any commands, but echo them instead. Display extra debug info')
    parser.add_argument('--init', action='store_true', default=False,
                        help='Re-initialize root and version')
    config = parser.parse_args()

    return config


def load(urlString, encoding="utf-8"):
    try:
        content = urllib.request.urlopen(urlString).read().decode(encoding)
    except Exception as e:
        print('Retrying download of url %s after exception: %s' % (urlString, e))
        content = urllib.request.urlopen(urlString).read().decode(encoding)
    return content


def configure_pgp(gpg_todo):
    print("Based on your Apache ID we'll lookup your key online\n"
          "and through this complete the 'gpg' prerequisite task.\n")
    gpg_state = gpg_todo.get_state()
    id = str(input("Please enter your Apache id: (ENTER=skip) "))
    if id.strip() == '':
        return False
    all_keys = load('https://home.apache.org/keys/group/lucene.asc')
    lines = all_keys.splitlines()
    keyid_linenum = None
    for idx, line in enumerate(lines):
        if line == 'ASF ID: %s' % id:
            keyid_linenum = idx+1
            break
    if keyid_linenum:
        keyid_line = lines[keyid_linenum]
        assert keyid_line.startswith('LDAP PGP key: ')
        gpg_id = keyid_line[14:].replace(" ", "")[-8:]
        print("Found gpg key id %s on file at Apache (https://home.apache.org/keys/group/lucene.asc)" % gpg_id)
    else:
        print(textwrap.dedent("""\
            Could not find your GPG key from Apache servers.
            Please make sure you have registered your key ID in
            id.apache.org, see links for more info."""))
        gpg_id = str(input("Enter your key ID manually, 8 last characters (ENTER=skip): "))
        if gpg_id.strip() == '':
            return False
        elif len(gpg_id) != 8:
            print("gpg id must be the last 8 characters of your key id")
        gpg_id = gpg_id.upper()
    try:
        res = run("gpg --list-secret-keys %s" % gpg_id)
        print("Found key %s on your private gpg keychain" % gpg_id)
        # Check rsa and key length >= 4096
        match = re.search(r'^sec +((rsa|dsa)(\d{4})) ', res)
        type = "(unknown)"
        length = -1
        if match:
            type = match.group(2)
            length = int(match.group(3))
        else:
            match = re.search(r'^sec +((\d{4})([DR])/.*?) ', res)
            if match:
                type = 'rsa' if match.group(3) == 'R' else 'dsa'
                length = int(match.group(2))
            else:
                print("Could not determine type and key size for your key")
                print("%s" % res)
                if not ask_yes_no("Is your key of type RSA and size >= 2048 (ideally 4096)? "):
                    print("Sorry, please generate a new key, add to KEYS and register with id.apache.org")
                    return False
        if not type == 'rsa':
            print("We strongly recommend RSA type key, your is '%s'. Consider generating a new key." % type.upper())
        if length < 2048:
            print("Your key has key length of %s. Cannot use < 2048, please generate a new key before doing the release" % length)
            return False
        if length < 4096:
            print("Your key length is < 4096, Please generate a stronger key.")
            print("Alternatively, follow instructions in http://www.apache.org/dev/release-signing.html#note")
            if not ask_yes_no("Have you configured your gpg to avoid SHA-1?"):
                print("Please either generate a strong key or reconfigure your client")
                return False
        print("Validated that your key is of type RSA and has a length >= 2048 (%s)" % length)
    except:
        print(textwrap.dedent("""\
            Key not found on your private gpg keychain. In order to sign the release you'll
            need to fix this, then try again"""))
        return False
    try:
        lines = run("gpg --check-signatures %s" % gpg_id).splitlines()
        sigs = 0
        apache_sigs = 0
        for line in lines:
            if line.startswith("sig") and not gpg_id in line:
                sigs += 1
                if '@apache.org' in line:
                    apache_sigs += 1
        print("Your key has %s signatures, of which %s are by committers (@apache.org address)" % (sigs, apache_sigs))
        if apache_sigs < 1:
            print(textwrap.dedent("""\
                Your key is not signed by any other committer. 
                Please review http://www.apache.org/dev/openpgp.html#apache-wot
                and make sure to get your key signed until next time.
                You may want to run 'gpg --refresh-keys' to refresh your keychain."""))
        uses_apacheid = is_code_signing_key = False
        for line in lines:
            if line.startswith("uid") and "%s@apache" % id in line:
                uses_apacheid = True
                if 'CODE SIGNING KEY' in line.upper():
                    is_code_signing_key = True
        if not uses_apacheid:
            print("WARNING: Your key should use your apache-id email address, see http://www.apache.org/dev/release-signing.html#user-id")
        if not is_code_signing_key:
            print("WARNING: You code signing key should be labeled 'CODE SIGNING KEY', see http://www.apache.org/dev/release-signing.html#key-comment")
    except Exception as e:
        print("Could not check signatures of your key: %s" % e)

    download_keys()
    keys_text = file_to_string(os.path.join(state.config_path, "KEYS"))
    if gpg_id in keys_text or "%s %s" % (gpg_id[:4], gpg_id[-4:]) in keys_text:
        print("Found your key ID in official KEYS file. KEYS file is not cached locally.")
    else:
        print(textwrap.dedent("""\
            Could not find your key ID in official KEYS file.
            Please make sure it is added to https://dist.apache.org/repos/dist/release/lucene/KEYS
            and committed to svn. Then re-try this initialization"""))
        if not ask_yes_no("Do you want to continue without fixing KEYS file? (not recommended) "):
            return False

    gpg_state['apache_id'] = id
    gpg_state['gpg_key'] = gpg_id
    return True


def pause(fun=None):
    if fun:
        fun()
    input("\nPress ENTER to continue...")


# Custom classes for ConsoleMenu, to make menu texts dynamic
# Needed until https://github.com/aegirhall/console-menu/pull/25 is released
# See https://pypi.org/project/console-menu/ for other docs

class UpdatableConsoleMenu(ConsoleMenu):

    def __repr__(self):
        return "%s: %s. %d items" % (self.get_title(), self.get_subtitle(), len(self.items))

    def draw(self):
        """
        Refreshes the screen and redraws the menu. Should be called whenever something changes that needs to be redrawn.
        """
        self.screen.printf(self.formatter.format(title=self.get_title(), subtitle=self.get_subtitle(), items=self.items,
                                                 prologue_text=self.get_prologue_text(), epilogue_text=self.get_epilogue_text()))

    # Getters to get text in case method reference
    def get_title(self):
        return self.title() if callable(self.title) else self.title

    def get_subtitle(self):
        return self.subtitle() if callable(self.subtitle) else self.subtitle

    def get_prologue_text(self):
        return self.prologue_text() if callable(self.prologue_text) else self.prologue_text

    def get_epilogue_text(self):
        return self.epilogue_text() if callable(self.epilogue_text) else self.epilogue_text


class UpdatableSubmenuItem(SubmenuItem):
    def __init__(self, text, submenu, menu=None, should_exit=False):
        """
        :ivar ConsoleMenu self.submenu: The submenu to be opened when this item is selected
        """
        super(SubmenuItem, self).__init__(text=text, menu=menu, should_exit=should_exit)

        self.submenu = submenu
        if menu:
            self.get_submenu().parent = menu

    def show(self, index):
        return "%2d - %s" % (index + 1, self.get_text())

    # Getters to get text in case method reference
    def get_text(self):
        return self.text() if callable(self.text) else self.text

    def set_menu(self, menu):
        """
        Sets the menu of this item.
        Should be used instead of directly accessing the menu attribute for this class.

        :param ConsoleMenu menu: the menu
        """
        self.menu = menu
        self.get_submenu().parent = menu

    def action(self):
        """
        This class overrides this method
        """
        self.get_submenu().start()

    def clean_up(self):
        """
        This class overrides this method
        """
        self.get_submenu().join()
        self.menu.clear_screen()
        self.menu.resume()

    def get_return(self):
        """
        :return: The returned value in the submenu
        """
        return self.get_submenu().returned_value

    def get_submenu(self):
        """
        We unwrap the submenu variable in case it is a reference to a method that returns a submenu
        """
        return self.submenu if not callable(self.submenu) else self.submenu()


class UpdatableFunctionItem(FunctionItem):
    def show(self, index):
        return "%2d - %s" % (index + 1, self.get_text())

    # Getters to get text in case method reference
    def get_text(self):
        return self.text() if callable(self.text) else self.text


class MyScreen(Screen):
    def clear(self):
        return


class CustomExitItem(ExitItem):
    def show(self, index):
        return super(ExitItem, self).show(index)

    def get_return(self):
        return ""


def main():
    global state
    global dry_run
    global templates

    print("Lucene/Solr releaseWizard v%s" % getScriptVersion())
    c = parse_config()

    if c.dry:
        print("Entering dry-run mode where all commands will be echoed instead of executed")
        dry_run = True

    release_root = os.path.expanduser("~/.lucene-releases")
    if not load_rc() or c.init:
        print("Initializing")
        dir_ok = False
        root = str(input("Choose root folder: [~/.lucene-releases] "))
        if os.path.exists(root) and (not os.path.isdir(root) or not os.access(root, os.W_OK)):
            sys.exit("Root %s exists but is not a directory or is not writable" % root)
        if not root == '':
            if root.startswith("~/"):
                release_root = os.path.expanduser(root)
            else:
                release_root = os.path.abspath(root)
        if not os.path.exists(release_root):
            try:
                print("Creating release root %s" % release_root)
                os.makedirs(release_root)
            except Exception as e:
                sys.exit("Error while creating %s: %s" % (release_root, e))
        release_version = get_release_version()
    else:
        conf = load_rc()
        release_root = conf['root']
        if 'release_version' in conf:
            release_version = conf['release_version']
        else:
            release_version = get_release_version()
    store_rc(release_root, release_version)

    check_prerequisites()

    try:
        y = yaml.load(open(os.path.join(script_path, "releaseWizard.yaml"), "r"), Loader=yaml.Loader)
        templates = y.get('templates')
        todo_list = y.get('groups')
        state = ReleaseState(release_root, release_version, getScriptVersion())
        state.init_todos(bootstrap_todos(todo_list))
        state.load()
    except Exception as e:
        sys.exit("Failed initializing. %s" % e)

    state.save()

    # Smoketester requires JAVA11_HOME to point to Java11
    os.environ['JAVA_HOME'] = state.get_java_home()
    os.environ['JAVACMD'] = state.get_java_cmd()

    global lucene_news_file
    global solr_news_file
    lucene_news_file = os.path.join(state.get_website_git_folder(), 'content', 'core', 'core_news',
      "%s-%s-available.md" % (state.get_release_date_iso(), state.release_version.replace(".", "-")))
    solr_news_file = os.path.join(state.get_website_git_folder(), 'content', 'solr', 'solr_news',
      "%s-%s-available.md" % (state.get_release_date_iso(), state.release_version.replace(".", "-")))
    website_folder = state.get_website_git_folder()

    main_menu = UpdatableConsoleMenu(title="Lucene/Solr ReleaseWizard",
                            subtitle=get_releasing_text,
                            prologue_text="Welcome to the release wizard. From here you can manage the process including creating new RCs. "
                                          "All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.",
                            epilogue_text="® 2020 The Lucene/Solr project. Licensed under the Apache License 2.0\nScript version v%s)" % getScriptVersion(),
                            screen=MyScreen())

    todo_menu = UpdatableConsoleMenu(title=get_releasing_text,
                            subtitle=get_subtitle,
                            prologue_text=None,
                            epilogue_text=None,
                            screen=MyScreen())
    todo_menu.exit_item = CustomExitItem("Return")

    for todo_group in state.todo_groups:
        if todo_group.num_applies() >= 0:
            menu_item = todo_group.get_menu_item()
            menu_item.set_menu(todo_menu)
            todo_menu.append_item(menu_item)

    main_menu.append_item(UpdatableSubmenuItem(get_todo_menuitem_title, todo_menu, menu=main_menu))
    main_menu.append_item(UpdatableFunctionItem(get_start_new_rc_menu_title, start_new_rc))
    main_menu.append_item(UpdatableFunctionItem('Clear and restart current RC', state.clear_rc))
    main_menu.append_item(UpdatableFunctionItem("Clear all state, restart the %s release" % state.release_version, reset_state))
    main_menu.append_item(UpdatableFunctionItem('Start release for a different version', release_other_version))
    main_menu.append_item(UpdatableFunctionItem('Generate Asciidoc guide for this release', generate_asciidoc))
    # main_menu.append_item(UpdatableFunctionItem('Dump YAML', dump_yaml))
    main_menu.append_item(UpdatableFunctionItem('Help', help))

    main_menu.show()


sys.path.append(os.path.dirname(__file__))
current_git_root = os.path.abspath(
    os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir, os.path.pardir))

dry_run = False

major_minor = ['major', 'minor']
script_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(script_path)


def git_checkout_folder():
    return state.get_git_checkout_folder()


def tail_file(file, lines):
    bufsize = 8192
    fsize = os.stat(file).st_size
    with open(file) as f:
        if bufsize >= fsize:
            bufsize = fsize
        idx = 0
        while True:
            idx += 1
            seek_pos = fsize - bufsize * idx
            if seek_pos < 0:
                seek_pos = 0
            f.seek(seek_pos)
            data = []
            data.extend(f.readlines())
            if len(data) >= lines or f.tell() == 0 or seek_pos == 0:
                if not seek_pos == 0:
                    print("Tailing last %d lines of file %s" % (lines, file))
                print(''.join(data[-lines:]))
                break


def run_with_log_tail(command, cwd, logfile=None, tail_lines=10, tee=False, live=False, shell=None):
    fh = sys.stdout
    if logfile:
        logdir = os.path.dirname(logfile)
        if not os.path.exists(logdir):
            os.makedirs(logdir)
        fh = open(logfile, 'w')
    rc = run_follow(command, cwd, fh=fh, tee=tee, live=live, shell=shell)
    if logfile:
        fh.close()
        if not tee and tail_lines and tail_lines > 0:
            tail_file(logfile, tail_lines)
    return rc


def ask_yes_no(text):
    answer = None
    while answer not in ['y', 'n']:
        answer = str(input("\nQ: %s (y/n): " % text))
    print("\n")
    return answer == 'y'


def abbreviate_line(line, width):
    line = line.rstrip()
    if len(line) > width:
        line = "%s.....%s" % (line[:(width / 2 - 5)], line[-(width / 2):])
    else:
        line = "%s%s" % (line, " " * (width - len(line) + 2))
    return line


def print_line_cr(line, linenum, stdout=True, tee=False):
    if not tee:
        if not stdout:
            print("[line %s] %s" % (linenum, abbreviate_line(line, 80)), end='\r')
    else:
        if line.endswith("\r"):
            print(line.rstrip(), end='\r')
        else:
            print(line.rstrip())


def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=None):
    doShell = '&&' in command or '&' in command or shell is not None
    if not doShell and not isinstance(command, list):
        command = shlex.split(command)
    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd,
                               universal_newlines=True, bufsize=0, close_fds=True, shell=doShell)
    lines_written = 0

    fl = fcntl.fcntl(process.stdout, fcntl.F_GETFL)
    fcntl.fcntl(process.stdout, fcntl.F_SETFL, fl | os.O_NONBLOCK)

    flerr = fcntl.fcntl(process.stderr, fcntl.F_GETFL)
    fcntl.fcntl(process.stderr, fcntl.F_SETFL, flerr | os.O_NONBLOCK)

    endstdout = endstderr = False
    errlines = []
    while not (endstderr and endstdout):
        lines_before = lines_written
        if not endstdout:
            try:
                if live:
                    chars = process.stdout.read()
                    if chars == '' and process.poll() is not None:
                        endstdout = True
                    else:
                        fh.write(chars)
                        fh.flush()
                        if '\n' in chars:
                            lines_written += 1
                else:
                    line = process.stdout.readline()
                    if line == '' and process.poll() is not None:
                        endstdout = True
                    else:
                        fh.write("%s\n" % line.rstrip())
                        fh.flush()
                        lines_written += 1
                        print_line_cr(line, lines_written, stdout=(fh == sys.stdout), tee=tee)

            except Exception as ioe:
                pass
        if not endstderr:
            try:
                if live:
                    chars = process.stderr.read()
                    if chars == '' and process.poll() is not None:
                        endstderr = True
                    else:
                        fh.write(chars)
                        fh.flush()
                        if '\n' in chars:
                            lines_written += 1
                else:
                    line = process.stderr.readline()
                    if line == '' and process.poll() is not None:
                        endstderr = True
                    else:
                        errlines.append("%s\n" % line.rstrip())
                        lines_written += 1
                        print_line_cr(line, lines_written, stdout=(fh == sys.stdout), tee=tee)
            except Exception as e:
                pass

        if not lines_written > lines_before:
            # if no output then sleep a bit before checking again
            time.sleep(0.1)

    print(" " * 80)
    rc = process.poll()
    if len(errlines) > 0:
        for line in errlines:
            fh.write("%s\n" % line.rstrip())
            fh.flush()
    return rc


def is_windows():
    return platform.system().startswith("Win")

def is_mac():
    return platform.system().startswith("Darwin")

def is_linux():
    return platform.system().startswith("Linux")

class Commands(SecretYamlObject):
    yaml_tag = u'!Commands'
    hidden_fields = ['todo_id']
    cmd_continuation_char = "^" if is_windows() else "\\"
    def __init__(self, root_folder, commands_text=None, commands=None, logs_prefix=None, run_text=None, enable_execute=None,
                 confirm_each_command=None, env=None, vars=None, todo_id=None, remove_files=None):
        self.root_folder = root_folder
        self.commands_text = commands_text
        self.vars = vars
        self.env = env
        self.run_text = run_text
        self.remove_files = remove_files
        self.todo_id = todo_id
        self.logs_prefix = logs_prefix
        self.enable_execute = enable_execute
        self.confirm_each_command = confirm_each_command
        self.commands = commands
        for c in self.commands:
            c.todo_id = todo_id

    @classmethod
    def from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node, deep = True)
        return Commands(**fields)

    def run(self):
        root = self.get_root_folder()

        if self.commands_text:
            print(self.get_commands_text())
        if self.env:
            for key in self.env:
                val = self.jinjaify(self.env[key])
                os.environ[key] = val
                if is_windows():
                    print("\n  SET %s=%s" % (key, val))
                else:
                    print("\n  export %s=%s" % (key, val))
        print(abbreviate_homedir("\n  cd %s" % root))
        commands = ensure_list(self.commands)
        for cmd in commands:
            for line in cmd.display_cmd():
                print("  %s" % line)
        print()
        if not self.enable_execute is False:
            if self.run_text:
                print("\n%s\n" % self.get_run_text())
            if len(commands) > 1:
                if not self.confirm_each_command is False:
                    print("You will get prompted before running each individual command.")
                else:
                    print(
                        "You will not be prompted for each command but will see the ouput of each. If one command fails the execution will stop.")
            success = True
            if ask_yes_no("Do you want me to run these commands now?"):
                if self.remove_files:
                    for _f in ensure_list(self.get_remove_files()):
                        f = os.path.join(root, _f)
                        if os.path.exists(f):
                            filefolder = "File" if os.path.isfile(f) else "Folder"
                            if ask_yes_no("%s %s already exists. Shall I remove it now?" % (filefolder, f)) and not dry_run:
                                if os.path.isdir(f):
                                    shutil.rmtree(f)
                                else:
                                    os.remove(f)
                index = 0
                log_folder = self.logs_prefix if len(commands) > 1 else None
                for cmd in commands:
                    index += 1
                    if len(commands) > 1:
                        log_prefix = "%02d_" % index
                    else:
                        log_prefix = self.logs_prefix if self.logs_prefix else ''
                    if not log_prefix[-1:] == '_':
                        log_prefix += "_"
                    cwd = root
                    if cmd.cwd:
                        cwd = os.path.join(root, cmd.cwd)
                    folder_prefix = ''
                    if cmd.cwd:
                        folder_prefix = cmd.cwd + "_"
                    if self.confirm_each_command is False or len(commands) == 1 or ask_yes_no("Shall I run '%s' in folder '%s'" % (cmd, cwd)):
                        if self.confirm_each_command is False:
                            print("------------\nRunning '%s' in folder '%s'" % (cmd, cwd))
                        logfilename = cmd.logfile
                        logfile = None
                        cmd_to_run = "%s%s" % ("echo Dry run, command is: " if dry_run else "", cmd.get_cmd())
                        if cmd.redirect:
                            try:
                                out = run(cmd_to_run, cwd=cwd)
                                mode = 'a' if cmd.redirect_append is True else 'w'
                                with open(os.path.join(root, cwd, cmd.get_redirect()), mode) as outfile:
                                    outfile.write(out)
                                    outfile.flush()
                                print("Wrote %s bytes to redirect file %s" % (len(out), cmd.get_redirect()))
                            except Exception as e:
                                print("Command %s failed: %s" % (cmd_to_run, e))
                                success = False
                                break
                        else:
                            if not cmd.stdout:
                                if not log_folder:
                                    log_folder = os.path.join(state.get_rc_folder(), "logs")
                                elif not os.path.isabs(log_folder):
                                    log_folder = os.path.join(state.get_rc_folder(), "logs", log_folder)
                                if not logfilename:
                                    logfilename = "%s.log" % re.sub(r"\W", "_", cmd.get_cmd())
                                logfile = os.path.join(log_folder, "%s%s%s" % (log_prefix, folder_prefix, logfilename))
                                if cmd.tee:
                                    print("Output of command will be printed (logfile=%s)" % logfile)
                                elif cmd.live:
                                    print("Output will be shown live byte by byte")
                                    logfile = None
                                else:
                                    print("Wait until command completes... Full log in %s\n" % logfile)
                                if cmd.comment:
                                    print("# %s\n" % cmd.get_comment())
                            start_time = time.time()
                            returncode = run_with_log_tail(cmd_to_run, cwd, logfile=logfile, tee=cmd.tee, tail_lines=25,
                                                           live=cmd.live, shell=cmd.shell)
                            elapsed = time.time() - start_time
                            if not returncode == 0:
                                if cmd.should_fail:
                                    print("Command failed, which was expected")
                                    success = True
                                else:
                                    print("WARN: Command %s returned with error" % cmd.get_cmd())
                                    success = False
                                    break
                            else:
                                if cmd.should_fail and not dry_run:
                                    print("Expected command to fail, but it succeeded.")
                                    success = False
                                    break
                                else:
                                    if elapsed > 30:
                                        print("Command completed in %s seconds" % elapsed)
            if not success:
                print("WARNING: One or more commands failed, you may want to check the logs")
            return success

    def get_root_folder(self):
        return self.jinjaify(self.root_folder)

    def get_commands_text(self):
        return self.jinjaify(self.commands_text)

    def get_run_text(self):
        return self.jinjaify(self.run_text)

    def get_remove_files(self):
        return self.jinjaify(self.remove_files)

    def get_vars(self):
        myvars = {}
        if self.vars:
            for k in self.vars:
                val = self.vars[k]
                if callable(val):
                    myvars[k] = expand_jinja(val(), vars=myvars)
                else:
                    myvars[k] = expand_jinja(val, vars=myvars)
        return myvars

    def jinjaify(self, data, join=False):
        if not data:
            return None
        v = self.get_vars()
        if self.todo_id:
            v.update(state.get_todo_by_id(self.todo_id).get_vars())
        if isinstance(data, list):
            if join:
                return expand_jinja(" ".join(data), v)
            else:
                res = []
                for rf in data:
                    res.append(expand_jinja(rf, v))
                return res
        else:
            return expand_jinja(data, v)


def abbreviate_homedir(line):
    if is_windows():
        if 'HOME' in os.environ:
            return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1%HOME%", line)
        elif 'USERPROFILE' in os.environ:
            return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1%USERPROFILE%", line)
    else:
        return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1~", line)


class Command(SecretYamlObject):
    yaml_tag = u'!Command'
    hidden_fields = ['todo_id']
    def __init__(self, cmd, cwd=None, stdout=None, logfile=None, tee=None, live=None, comment=None, vars=None,
                 todo_id=None, should_fail=None, redirect=None, redirect_append=None, shell=None):
        self.cmd = cmd
        self.cwd = cwd
        self.comment = comment
        self.logfile = logfile
        self.vars = vars
        self.tee = tee
        self.live = live
        self.stdout = stdout
        self.should_fail = should_fail
        self.shell = shell
        self.todo_id = todo_id
        self.redirect_append = redirect_append
        self.redirect = redirect
        if tee and stdout:
            self.stdout = None
            print("Command %s specifies 'tee' and 'stdout', using only 'tee'" % self.cmd)
        if live and stdout:
            self.stdout = None
            print("Command %s specifies 'live' and 'stdout', using only 'live'" % self.cmd)
        if live and tee:
            self.tee = None
            print("Command %s specifies 'tee' and 'live', using only 'live'" % self.cmd)
        if redirect and (tee or stdout or live):
            self.tee = self.stdout = self.live = None
            print("Command %s specifies 'redirect' and other out options at the same time. Using redirect only" % self.cmd)

    @classmethod
    def from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node, deep = True)
        return Command(**fields)

    def get_comment(self):
        return self.jinjaify(self.comment)

    def get_redirect(self):
        return self.jinjaify(self.redirect)

    def get_cmd(self):
        return self.jinjaify(self.cmd, join=True)

    def get_vars(self):
        myvars = {}
        if self.vars:
            for k in self.vars:
                val = self.vars[k]
                if callable(val):
                    myvars[k] = expand_jinja(val(), vars=myvars)
                else:
                    myvars[k] = expand_jinja(val, vars=myvars)
        return myvars

    def __str__(self):
        return self.get_cmd()

    def jinjaify(self, data, join=False):
        v = self.get_vars()
        if self.todo_id:
            v.update(state.get_todo_by_id(self.todo_id).get_vars())
        if isinstance(data, list):
            if join:
                return expand_jinja(" ".join(data), v)
            else:
                res = []
                for rf in data:
                    res.append(expand_jinja(rf, v))
                return res
        else:
            return expand_jinja(data, v)

    def display_cmd(self):
        lines = []
        pre = post = ''
        if self.comment:
            if is_windows():
                lines.append("REM %s" % self.get_comment())
            else:
                lines.append("# %s" % self.get_comment())
        if self.cwd:
            lines.append("pushd %s" % self.cwd)
        redir = "" if self.redirect is None else " %s %s" % (">" if self.redirect_append is None else ">>" , self.get_redirect())
        line = "%s%s" % (expand_multiline(self.get_cmd(), indent=2), redir)
        # Print ~ or %HOME% rather than the full expanded homedir path
        line = abbreviate_homedir(line)
        lines.append(line)
        if self.cwd:
            lines.append("popd")
        return lines

class UserInput(SecretYamlObject):
    yaml_tag = u'!UserInput'

    def __init__(self, name, prompt, type=None):
        self.type = type
        self.prompt = prompt
        self.name = name

    @classmethod
    def from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node, deep = True)
        return UserInput(**fields)

    def run(self, dict=None):
        correct = False
        while not correct:
            try:
                result = str(input("%s : " % self.prompt))
                if self.type and self.type == 'int':
                    result = int(result)
                correct = True
            except Exception as e:
                print("Incorrect input: %s, try again" % e)
                continue
            if dict:
                dict[self.name] = result
            return result


def create_ical(todo):
    if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):
        c = Calendar()
        e = Event()
        e.name = "Lucene/Solr %s vote ends" % state.release_version
        e.begin = vote_close_72h_date()
        e.description = "Remember to sum up votes and continue release :)"
        c.events.add(e)
        ics_file = os.path.join(state.get_rc_folder(), 'vote_end.ics')
        with open(ics_file, 'w') as my_file:
            my_file.writelines(c)
        open_file(ics_file)
    return True


today = datetime.utcnow().date()
sundays = {(today + timedelta(days=x)): 'Sunday' for x in range(10) if (today + timedelta(days=x)).weekday() == 6}
y = datetime.utcnow().year
years = [y, y+1]
non_working = holidays.CA(years=years) + holidays.US(years=years) + holidays.England(years=years) \
              + holidays.DE(years=years) + holidays.NO(years=years) + holidays.IND(years=years) + holidays.RU(years=years)


def vote_close_72h_date():
    # Voting open at least 72 hours according to ASF policy
    return datetime.utcnow() + timedelta(hours=73)


def vote_close_72h_holidays():
    days = 0
    day_offset = -1
    holidays = []
    # Warn RM about major holidays coming up that should perhaps extend the voting deadline
    # Warning will be given for Sunday or a public holiday observed by 3 or more [CA, US, EN, DE, NO, IND, RU]
    while days < 3:
        day_offset += 1
        d = today + timedelta(days=day_offset)
        if not (d in sundays or (d in non_working and len(non_working[d]) >= 2)):
            days += 1
        else:
            if d in sundays:
                holidays.append("%s (Sunday)" % d)
            else:
                holidays.append("%s (%s)" % (d, non_working[d]))
    return holidays if len(holidays) > 0 else None


def prepare_announce_lucene(todo):
    if not os.path.exists(lucene_news_file):
        lucene_text = expand_jinja("(( template=announce_lucene ))")
        with open(lucene_news_file, 'w') as fp:
            fp.write(lucene_text)
        # print("Wrote Lucene announce draft to %s" % lucene_news_file)
    else:
        print("Draft already exist, not re-generating")
    return True

def prepare_announce_solr(todo):
    if not os.path.exists(solr_news_file):
        solr_text = expand_jinja("(( template=announce_solr ))")
        with open(solr_news_file, 'w') as fp:
            fp.write(solr_text)
        # print("Wrote Solr announce draft to %s" % solr_news_file)
    else:
        print("Draft already exist, not re-generating")
    return True


def set_java_home(version):
    os.environ['JAVA_HOME'] = state.get_java_home_for_version(version)
    os.environ['JAVACMD'] = state.get_java_cmd_for_version(version)


def load_lines(file, from_line=0):
    if os.path.exists(file):
        with open(file, 'r') as fp:
            return fp.readlines()[from_line:]
    else:
        return ["<Please paste the announcement text here>\n"]


if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print('Keyboard interrupt...exiting')