#!/usr/bin/env python """ Convert PEPs to (X)HTML fragments for Pyramid - courtesy of /F Usage: %(PROGRAM)s [options] [ ...] Options: -d , --destdir Specify the base destination directory for Pyramid files. Default: %(SERVER_DEST_DIR_BASE)s -f, --force Force the rebuilding of output files, regardless of modification times. -k, --keep-going Continue building past errors if possible. -q, --quiet Turn off verbose messages. -h, --help Print this help message and exit. The optional arguments ``peps`` are either pep numbers or .txt files. """ import sys import os import codecs import re import cgi import glob import getopt import errno import random import time import shutil REQUIRES = {'python': '2.2', 'docutils': '0.5'} PROGRAM = sys.argv[0] SERVER_DEST_DIR_BASE = ( '/data/ftp.python.org/pub/beta.python.org/build/data/dev/peps') RFCURL = 'http://www.faqs.org/rfcs/rfc%d.html' PEPCVSURL = 'http://svn.python.org/view/*checkout*/peps/trunk/pep-%04d.txt' PEPDIRURL = '/dev/peps/' PEPURL = PEPDIRURL + 'pep-%04d' PEPANCHOR = '%i' LOCALVARS = "Local Variables:" COMMENT = """""" # The generated HTML doesn't validate -- you cannot use
and

inside #
 tags.  But if I change that, the result doesn't look very nice...

fixpat = re.compile("((https?|ftp):[-_a-zA-Z0-9/.+~:?#$=&,]+)|(pep-\d+(.txt)?)|"
                    "(RFC[- ]?(?P\d+))|"
                    "(PEP\s+(?P\d+))|"
                    ".")

CONTENT_HTML = """\

' print >> outfile, '

%s

' % line.strip() need_pre = 1 elif not line.strip() and need_pre: continue else: # PEP 0 has some special treatment if basename == 'pep-0000.txt': parts = line.split() if len(parts) > 1 and re.match(r'\s*\d{1,4}', parts[1]): # This is a PEP summary line, which we need to hyperlink url = PEPURL % int(parts[1]) if need_pre: print >> outfile, '
'
                        need_pre = 0
                    print >> outfile, re.sub(
                        parts[1],
                        '%s' % (int(parts[1]),
                            parts[1]), line, 1),
                    continue
                elif parts and '@' in parts[-1]:
                    # This is a pep email address line, so filter it.
                    url = fixemail(parts[-1], pep)
                    if need_pre:
                        print >> outfile, '
'
                        need_pre = 0
                    print >> outfile, re.sub(
                        parts[-1], url, line, 1),
                    continue
            line = fixpat.sub(lambda x, c=inpath: fixanchor(c, x), line)
            if need_pre:
                print >> outfile, '
'
                need_pre = 0
            outfile.write(line)
    if not need_pre:
        print >> outfile, '
' return title docutils_settings = None """Runtime settings object used by Docutils. Can be set by the client application when this module is imported.""" def fix_rst_pep(inpath, input_lines, outfile): from docutils import core parts = core.publish_parts( source=''.join(input_lines), source_path=inpath, destination_path=outfile.name, reader_name='pep', parser_name='restructuredtext', writer_name='pep_html', settings=docutils_settings, # Allow Docutils traceback if there's an exception: settings_overrides={'traceback': 1}) outfile.write(parts['whole']) title = 'PEP %s -- %s' % (parts['pepnum'], parts['title'][0]) return title def get_pep_type(input_lines): """ Return the Content-Type of the input. "text/plain" is the default. Return ``None`` if the input is not a PEP. """ pep_type = None for line in input_lines: line = line.rstrip().lower() if not line: # End of the RFC 2822 header (first blank line). break elif line.startswith('content-type: '): pep_type = line.split()[1] or 'text/plain' break elif line.startswith('pep: '): # Default PEP type, used if no explicit content-type specified: pep_type = 'text/plain' return pep_type def get_input_lines(inpath): try: infile = codecs.open(inpath, 'r', 'utf-8') except IOError, e: if e.errno <> errno.ENOENT: raise print >> sys.stderr, 'Error: Skipping missing PEP file:', e.filename sys.stderr.flush() return None, None lines = infile.read().splitlines(1) # handles x-platform line endings infile.close() return lines def find_pep(pep_str): """Find the .txt file indicated by a cmd line argument""" if os.path.exists(pep_str): return pep_str num = int(pep_str) return "pep-%04d.txt" % num def make_html(inpath): input_lines = get_input_lines(inpath) pep_type = get_pep_type(input_lines) if pep_type is None: print >> sys.stderr, 'Error: Input file %s is not a PEP.' % inpath sys.stdout.flush() return None elif not PEP_TYPE_DISPATCH.has_key(pep_type): print >> sys.stderr, ('Error: Unknown PEP type for input file %s: %s' % (inpath, pep_type)) sys.stdout.flush() return None elif PEP_TYPE_DISPATCH[pep_type] == None: pep_type_error(inpath, pep_type) return None destDir, needSvn, pepnum = set_up_pyramid(inpath) outpath = os.path.join(destDir, 'body.html') if ( not settings.force_rebuild and (os.path.exists(outpath) and os.stat(inpath).st_mtime <= os.stat(outpath).st_mtime)): if settings.verbose: print "Skipping %s (outfile up to date)"%(inpath) return if settings.verbose: print inpath, "(%s)" % pep_type, "->", outpath sys.stdout.flush() outfile = codecs.open(outpath, "w", "utf-8") title = PEP_TYPE_DISPATCH[pep_type](inpath, input_lines, outfile) outfile.close() os.chmod(outfile.name, 0664) write_pyramid_index(destDir, title) # for PEP 0, copy body to parent directory as well if pepnum == '0000': shutil.copyfile(outpath, os.path.join(destDir, '..', 'body.html')) copy_aux_files(inpath, destDir) return outpath def set_up_pyramid(inpath): m = re.search(r'pep-(\d+)\.', inpath) if not m: print >>sys.stderr, "Can't find PEP number in file name." sys.exit(1) pepnum = m.group(1) destDir = os.path.join(settings.dest_dir_base, 'pep-%s' % pepnum) needSvn = 0 if not os.path.exists(destDir): needSvn = 1 os.makedirs(destDir) # write content.html foofilename = os.path.join(destDir, 'content.html') fp = codecs.open(foofilename, 'w', 'utf-8') fp.write(CONTENT_HTML) fp.close() os.chmod(foofilename, 0664) # write content.yml foofilename = os.path.join(destDir, 'content.yml') fp = codecs.open(foofilename, 'w', 'utf-8') fp.write(CONTENT_YML) os.chmod(foofilename, 0664) return destDir, needSvn, pepnum def write_pyramid_index(destDir, title): filename = os.path.join(destDir, 'index.yml') fp = codecs.open(filename, 'w', 'utf-8') title = title.replace('\\', '\\\\') # Escape existing backslashes fp.write(INDEX_YML % title.replace('"', '\\"')) fp.close() os.chmod(filename, 0664) def copy_aux_files(pep_path, dest_dir): """ Copy auxiliary files whose names match 'pep-XXXX-*.*'. """ dirname, pepname = os.path.split(pep_path) base, ext = os.path.splitext(pepname) files = glob.glob(os.path.join(dirname, base) + '-*.*') for path in files: filename = os.path.basename(path) dest_path = os.path.join(dest_dir, filename) print '%s -> %s' % (path, dest_path) shutil.copy(path, dest_path) PEP_TYPE_DISPATCH = {'text/plain': fixfile, 'text/x-rst': fix_rst_pep} PEP_TYPE_MESSAGES = {} def check_requirements(): # Check Python: try: from email.Utils import parseaddr except ImportError: PEP_TYPE_DISPATCH['text/plain'] = None PEP_TYPE_MESSAGES['text/plain'] = ( 'Python %s or better required for "%%(pep_type)s" PEP ' 'processing; %s present (%%(inpath)s).' % (REQUIRES['python'], sys.version.split()[0])) # Check Docutils: try: import docutils except ImportError: PEP_TYPE_DISPATCH['text/x-rst'] = None PEP_TYPE_MESSAGES['text/x-rst'] = ( 'Docutils not present for "%(pep_type)s" PEP file %(inpath)s. ' 'See README.txt for installation.') else: installed = [int(part) for part in docutils.__version__.split('.')] required = [int(part) for part in REQUIRES['docutils'].split('.')] if installed < required: PEP_TYPE_DISPATCH['text/x-rst'] = None PEP_TYPE_MESSAGES['text/x-rst'] = ( 'Docutils must be reinstalled for "%%(pep_type)s" PEP ' 'processing (%%(inpath)s). Version %s or better required; ' '%s present. See README.txt for installation.' % (REQUIRES['docutils'], docutils.__version__)) def pep_type_error(inpath, pep_type): print >> sys.stderr, 'Error: ' + PEP_TYPE_MESSAGES[pep_type] % locals() sys.stdout.flush() def build_peps(args=None): if args: filenames = pep_filename_generator(args) else: # do them all filenames = glob.glob("pep-*.txt") filenames.sort() for filename in filenames: try: make_html(filename) except (KeyboardInterrupt, SystemExit): raise except: print "While building PEPs: %s" % filename if settings.keep_going: ee, ev, et = sys.exc_info() traceback.print_exception(ee, ev, et, file=sys.stdout) print "--keep-going/-k specified, continuing" continue else: raise def pep_filename_generator(args): for pep in args: filename = find_pep(pep) yield filename def main(argv=None): check_requirements() if argv is None: argv = sys.argv[1:] try: opts, args = getopt.getopt( argv, 'hd:fkq', ['help', 'destdir=', 'force', 'keep-going', 'quiet']) except getopt.error, msg: usage(1, msg) for opt, arg in opts: if opt in ('-h', '--help'): usage(0) elif opt in ('-d', '--destdir'): settings.dest_dir_base = arg elif opt in ('-f', '--force'): settings.force_rebuild = True elif opt in ('-k', '--keep-going'): settings.force_rebuild = True elif opt in ('-q', '--quiet'): settings.verbose = False build_peps(args) if __name__ == "__main__": main()