From 2392eb9385c2866da10f7b2509279601de41b759 Mon Sep 17 00:00:00 2001 From: milde Date: Wed, 13 Jun 2012 23:27:03 +0000 Subject: [PATCH] Make tools/ compatible with both, Python 2 and 3 without 2to3-conversion. git-svn-id: https://docutils.svn.sourceforge.net/svnroot/docutils/trunk/docutils@7442 929543f6-e4f2-0310-98a6-ba3bd3dd1d04 --- HISTORY.txt | 6 +- README.txt | 165 ++++++++++++++++++++++++------------------ setup.cfg | 4 + setup.py | 12 ++- tools/buildhtml.py | 13 ++-- tools/dev/create_unimap.py | 39 ++++++---- tools/dev/profile_docutils.py | 10 +-- tools/dev/unicode2rstsubs.py | 25 ++++--- tools/quicktest.py | 22 +++--- tools/test/test_buildhtml.py | 31 ++++++-- 10 files changed, 194 insertions(+), 133 deletions(-) diff --git a/HISTORY.txt b/HISTORY.txt index ebee0ce61..445384533 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -25,9 +25,13 @@ Changes Since 0.9 * docutils/test/ - Make tests independent from the location of the ``test/`` directory. - - Use converted sources from the ``build/`` directory for tests under + - Use converted sources (from the ``build/`` directory) for tests under Python 3. +* docutils/tools/ + + - Make tools compatible with both, Python 2 and 3 without 2to3-conversion. + * docutils/io.py - Fix writing binary data to sys.stdout under Python 3 (allows diff --git a/README.txt b/README.txt index 7f942cbb1..57535acfb 100644 --- a/README.txt +++ b/README.txt @@ -14,45 +14,33 @@ Quick-Start =========== -This is for those who want to get up & running quickly. Read on for -complete details. +This is for those who want to get up & running quickly. -1. Get and install the latest release of Python, available from +1. Docutils requires Python (version 2.3 or later), available from - http://www.python.org/ + http://www.python.org/ - Docutils is compatible with Python versions from 2.3 up to 2.7 and - versions 3.1 and 3.2. (Support for Python 3 is new and might still - have some issues.) + See Requirements_ below for details. -2. Use the latest Docutils code. Get the code from Subversion or from - the snapshot: +2. Use the latest Docutils code. Get the code from the `Subversion + repository`_ or from the snapshot: - http://docutils.svn.sourceforge.net/viewvc/docutils/trunk/docutils/?view=tar + http://docutils.svn.sourceforge.net/viewvc/docutils/trunk/docutils/?view=tar See `Releases & Snapshots`_ below for details. 3. Unpack the tarball in a temporary directory (**not** directly in - Python's ``site-packages``) and run ``setup.py install`` or - ``install.py`` with admin rights. On Windows systems it may be - sufficient to double-click ``install.py``. On Unix or Mac OS X, - type:: - - su - (enter admin password) - ./setup.py install - - Docutils will only work with Python 3, if installed with a Python - version >= 3. If your default Python version is 2.x, also call - ``python3 setup.py install`` from the temporary directory. + Python's ``site-packages``), go to the directory created by expanding + the archive, and run ``setup.py install`` with admin rights. On + Windows systems it may be sufficient to double-click ``install.py``. + See Installation_ below for details. -4. Use a front-end tool from the "tools" subdirectory of the same - directory as in step 3. For example:: +4. Use the front-end scripts to convert reStructuredText documents. + Try for example:: - cd tools - ./rst2html.py ../FAQ.txt ../FAQ.html (Unix) - python rst2html.py ..\FAQ.txt ..\FAQ.html (Windows) + rst2html.py FAQ.txt FAQ.html (Unix) + python tools/rst2html.py FAQ.txt FAQ.html (Windows) See Usage_ below for details. @@ -103,8 +91,9 @@ changes being committed to the repository): * Snapshot of the Sandbox (experimental, contributed code): http://docutils.svn.sourceforge.net/viewvc/docutils/trunk/sandbox/?view=tar -To keep up to date on the latest developments, download fresh copies -of the snapshots regularly. (There's also the `Subversion repository`_.) +To keep up to date on the latest developments, download fresh copies of +the snapshots regularly or use a working copy of the +`Subversion repository`_. .. _Subversion repository: docs/dev/repository.html @@ -112,21 +101,48 @@ of the snapshots regularly. (There's also the `Subversion repository`_.) Requirements ============ -To run the code, Python 2.3 or later must already be installed. -Python is available from -http://www.python.org/. +To run the code, Python_ must be installed. +Docutils is compatible with Python versions from 2.3 up to 2.7 and +versions 3.1 and 3.2 (cf. `Python 3 compatibility`_). Docutils uses the following packages for enhanced functionality, if they are installed: -* The `Python Imaging Library`, or PIL, is used for some image +* The `Python Imaging Library`_, or PIL, is used for some image manipulation operations. * The `Pygments`_ syntax highlighter is used for content of `code` directives and roles. +.. _Python: http://www.python.org/. .. _Python Imaging Library: http://www.pythonware.com/products/pil/ -.. _pygments: http://pygments.org/ +.. _Pygments: http://pygments.org/ + + +Python 3 compatibility +---------------------- + +The Docutils codebase is written for Python 2 and uses "on-demand" +translation for `porting to Python 3`_. + +* The `setup.py` script generates Python 3 compatible sources in + ``build/`` and tests in ``tests3/`` sub-directories during + installation_ with Python 3. + +* The scripts in the ``tools/`` sub-directory work with all supported + Python versions without conversion. + +* To convert the sources without installing (e.g. for testing), run + ``python3 setup.py build``. + +* When editing the source, do changes on the Python 2 versions of the + files and re-run the build command. + +Using Docutils with Python 3.x is less tested and might still have some +issues. + +.. _porting to Python 3: http://docs.python.org/py3k/howto/pyporting.html + Project Files & Directories =========================== @@ -180,6 +196,12 @@ Project Files & Directories if you're planning to modify it. See `Running the Test Suite`_ below. +Generated directories when installing under Python 3: + +* build: Converted sources. + +* test3: Converted tests. + Installation ============ @@ -199,19 +221,26 @@ GNU/Linux, BSDs, Unix, Mac OS X, etc. cd -3. Install the package:: +3. Install the package (you may need root permissions to complete this + step):: + su + (enter admin password) python setup.py install If the python executable isn't on your path, you'll have to specify - the complete path, such as /usr/local/bin/python. You may need - root permissions to complete this step. + the complete path, such as ``/usr/local/bin/python``. - To install for a specific python version, use this version in the + To install for a specific Python version, use this version in the setup call, e.g. :: python3.1 setup.py install + To install for different Python versions, repeat step 3 for every + required version. The last installed version will be used in the + `shebang line`_ of the ``rst2*.py`` wrapper scripts. + + .. _shebang line: http://en.wikipedia.org/wiki/Shebang_%28Unix%29 Windows ------- @@ -233,23 +262,37 @@ following: To install for a specific python version, specify the Python executable for this version. -Developing under Python 3 -------------------------- + To install for different Python versions, repeat step 3 for every + required version. -Under Python 3, installing with ``setup.py`` converts the source to Python 3 -compatible code before installing. If you want to test or develop Docutils, -also run ``python3 setup.py build``. This will generate Python 3 compatible -sources, in the ``build/`` sub-directory, tests in ``tests3/``, and -developer tools in ``tools3``. +Optional steps: -Do changes on the Python 2 versions of the sources and re-run the build -command. This works incrementally, so if you change one file it will only -reconvert that file the next time you run setup.py build. +* `running the test suite`_ + +* `converting the documentation`_ Usage ===== +There are many front-end tools in the unpacked "tools" subdirectory. +Installation under Unix places copies in the PATH. +You may want to begin with the "rst2html.py" front-end tool. Most +tools take up to two arguments, the source path and destination path, +with STDIN and STDOUT being the defaults. Use the "--help" option to +the front-end tools for details on options and arguments. See +Docutils Front-End Tools (``docs/user/tools.txt``) for full documentation. + +The package modules are continually growing and evolving. The +``docutils.statemachine`` module is usable independently. It contains +extensive inline documentation (in reStructuredText format of course). + +Contributions are welcome! + + +Converting the documentation +============================ + After unpacking and installing the Docutils package, the following shell commands will generate HTML for all included documentation:: @@ -270,33 +313,18 @@ Alternatively:: tools/buildhtml.py --config=tools/docutils.conf (Unix) python tools\buildhtml.py --config=tools\docutils.conf (Windows) -With Python 3, call:: - - build//tools/buildhtml.py --config=tools/docutils.conf - Some files may generate system messages (warnings and errors). The ``docs/user/rst/demo.txt`` file (under the archive directory) contains five intentional errors. (They test the error reporting mechanism!) -There are many front-end tools in the unpacked "tools" subdirectory. -You may want to begin with the "rst2html.py" front-end tool. Most -tools take up to two arguments, the source path and destination path, -with STDIN and STDOUT being the defaults. Use the "--help" option to -the front-end tools for details on options and arguments. See -Docutils Front-End Tools (``docs/user/tools.txt``) for full documentation. - -The package modules are continually growing and evolving. The -``docutils.statemachine`` module is usable independently. It contains -extensive inline documentation (in reStructuredText format of course). - -Contributions are welcome! - Running the Test Suite ====================== -To run the entire test suite, after installation_ open a shell and use -the following commands:: +The test suite is documented in `Docutils Testing`_ (docs/dev/testing.txt). + +To run the entire test suite, open a shell and use the following +commands:: cd /test ./alltests.py @@ -340,9 +368,8 @@ Windows users type these commands:: cd ..\tools python quicktest.py --version -For Python 3, the path is ``tools3/quicktest.py``. - +.. _Docutils Testing: http://docutils.sourceforge.net/docs/dev/testing.html .. _open a bug report: http://sourceforge.net/tracker/?group_id=38414&atid=422030 .. _send email: mailto:docutils-users@lists.sourceforge.net diff --git a/setup.cfg b/setup.cfg index 3ca547d21..5460e0e6a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,3 +8,7 @@ doc_files = BUGS.txt THANKS.txt docs/ licenses/ + +# [build] +# executable = /usr/bin/env python +# Uncomment to keep unchanged in the "shebang line" of front-end scripts diff --git a/setup.py b/setup.py index 30db1b93a..8d7d0d625 100755 --- a/setup.py +++ b/setup.py @@ -28,11 +28,11 @@ if sys.version_info >= (3,): class copy_build_py_2to3(build_py_2to3): """Copy/convert Python source files in given directories recursively. - Build py3k versions of the modules and packages. Also copy - 'tools/' and 'test/' dirs and run 2to3 on *.py files. + Build py3k versions of the modules and packages. + Also copy 'test/' suite and run 2to3 on *.py files. """ manifest_in = """\ - exclude *.pyc *~ .DS_Store rst2*.py rstpep2html.py + exclude *.pyc *~ .DS_Store recursive-exclude * *.pyc *~ .DS_Store recursive-exclude functional/output * include functional/output/README.txt @@ -45,11 +45,9 @@ if sys.version_info >= (3,): """ def run(self): build_py_2to3.run(self) - print("copying aux dirs") + print("copy/convert test suite") loglevel = log.set_threshold(log.ERROR) - for source in ['tools', 'test']: - dest = source + '3' - copydir_run_2to3(source, dest, template=self.manifest_in) + copydir_run_2to3('test', 'test3', template=self.manifest_in) log.set_threshold(loglevel) diff --git a/tools/buildhtml.py b/tools/buildhtml.py index 954c53a15..74dd69fb1 100755 --- a/tools/buildhtml.py +++ b/tools/buildhtml.py @@ -200,13 +200,13 @@ class Builder: settings = self.get_settings('', directory) errout = ErrorOutput(encoding=settings.error_encoding) if settings.prune and (os.path.abspath(directory) in settings.prune): - print >>errout, ('/// ...Skipping directory (pruned): %s' % - directory) + errout.write('/// ...Skipping directory (pruned): %s\n' % + directory) sys.stderr.flush() del subdirectories[:] return if not self.initial_settings.silent: - print >>errout, '/// Processing directory: %s' % directory + errout.write('/// Processing directory: %s' % directory) sys.stderr.flush() # settings.ignore grows many duplicate entries as we recurse # if we add patterns in config files or on the command line. @@ -230,7 +230,7 @@ class Builder: settings._source = os.path.normpath(os.path.join(directory, name)) settings._destination = settings._source[:-4]+'.html' if not self.initial_settings.silent: - print >>errout, ' ::: Processing: %s' % name + errout.write(' ::: Processing: %s\n' % name) sys.stderr.flush() try: if not settings.dry_run: @@ -240,8 +240,9 @@ class Builder: parser_name='restructuredtext', writer_name=pub_struct.writer_name, settings=settings) - except ApplicationError, error: - print >>errout, ' %s' % ErrorString(error) + except ApplicationError: + error = sys.exc_info()[1] # get exception in Python <2.6 and 3.x + errout.write(' %s\n' % ErrorString(error)) if __name__ == "__main__": diff --git a/tools/dev/create_unimap.py b/tools/dev/create_unimap.py index 0f733c69f..9fb75bcc7 100755 --- a/tools/dev/create_unimap.py +++ b/tools/dev/create_unimap.py @@ -13,8 +13,15 @@ from xml.dom import minidom import sys import pprint +if sys.version_info >= (3,0): + unicode = str +else: + bytes = str + chr = unichr + + def w(s): - if isinstance(s, unicode): + if sys.version_info >= (3,0) and isinstance(s, unicode): s = s.encode('utf8') sys.stdout.write(s) @@ -39,9 +46,9 @@ class Visitor: continue latex_code = n.childNodes[0].nodeValue.encode('ascii').strip() if node.attributes['mode'].value == 'math': - math_map[unichr(int(code))] = '$%s$' % latex_code + math_map[chr(int(code))] = '$%s$' % latex_code else: - text_map[unichr(int(code))] = '{%s}' % latex_code + text_map[chr(int(code))] = '{%s}' % latex_code def call_visitor(node, visitor=Visitor()): if isinstance(node, minidom.Text): @@ -63,16 +70,16 @@ unicode_map.update(text_map) # Now unicode_map contains the text entries plus dollar-enclosed math # entries for those chars for which no text entry exists. -print '# $%s$' % 'Id' -print '# Author: Lea Wiemann ' -print '# Copyright: This file has been placed in the public domain.' -print -print '# This is a mapping of Unicode characters to LaTeX equivalents.' -print '# The information has been extracted from' -print '# , written by' -print '# David Carlisle and Sebastian Rahtz.' -print '#' -print '# The extraction has been done by the "create_unimap.py" script' -print '# located at .' -print -print 'unicode_map = %s' % pprint.pformat(unicode_map, indent=0) +print('# $%s$' % 'Id') +print('# Author: Lea Wiemann ') +print('# Copyright: This file has been placed in the public domain.') +print('') +print('# This is a mapping of Unicode characters to LaTeX equivalents.') +print('# The information has been extracted from') +print('# , written by') +print('# David Carlisle and Sebastian Rahtz.') +print('#') +print('# The extraction has been done by the "create_unimap.py" script') +print('# located at .') +print('') +print('unicode_map = %s' % pprint.pformat(unicode_map, indent=0)) diff --git a/tools/dev/profile_docutils.py b/tools/dev/profile_docutils.py index 90b083787..d31ba6fd0 100755 --- a/tools/dev/profile_docutils.py +++ b/tools/dev/profile_docutils.py @@ -8,25 +8,25 @@ import os.path import docutils.core import hotshot.stats -print 'Profiler started.' +print('Profiler started.') os.chdir(os.path.join(os.path.dirname(docutils.__file__), '..')) -print 'Profiling...' +print('Profiling...') prof = hotshot.Profile('docutils.prof') prof.runcall(docutils.core.publish_file, source_path='HISTORY.txt', destination_path='prof.HISTORY.html', writer_name='html') prof.close() -print 'Loading statistics...' +print('Loading statistics...') -print """ +print(""" stats = hotshot.stats.load('docutils.prof') stats.strip_dirs() stats.sort_stats('time') # 'cumulative'; 'calls' stats.print_stats(40) -""" +""") stats = hotshot.stats.load('docutils.prof') stats.strip_dirs() diff --git a/tools/dev/unicode2rstsubs.py b/tools/dev/unicode2rstsubs.py index 22ac07ecc..b4e65e109 100755 --- a/tools/dev/unicode2rstsubs.py +++ b/tools/dev/unicode2rstsubs.py @@ -27,12 +27,12 @@ import re from xml.parsers.expat import ParserCreate -usage_msg = """Usage: %s [unicode.xml]""" +usage_msg = """Usage: %s [unicode.xml]\n""" def usage(prog, status=0, msg=None): - print >>sys.stderr, usage_msg % prog + sys.stderr.write(usage_msg % prog) if msg: - print >>sys.stderr, msg + sys.stderr.write(msg + '\n') sys.exit(status) def main(argv=None): @@ -47,7 +47,10 @@ def main(argv=None): inpath = 'unicode.xml' if not os.path.isfile(inpath): usage(argv[0], 1, 'No such file: "%s".' % inpath) - infile = open(inpath) + if sys.version_info >= (3,0): + infile = open(inpath, mode='rb') + else: + infile = open(inpath) process(infile) def process(infile): @@ -130,7 +133,7 @@ class CharacterEntitySetExtractor: if not set: return if set not in self.sets: - print 'bad set: %r' % set + print('bad set: %r' % set) return entity = attributes['id'] assert (entity not in self.sets[set] @@ -159,7 +162,7 @@ class CharacterEntitySetExtractor: return name def write_sets(self): - sets = self.sets.keys() + sets = list(self.sets.keys()) sets.sort() for set_name in sets: self.write_set(set_name) @@ -170,8 +173,8 @@ class CharacterEntitySetExtractor: else: outname = set_name + '.txt' outfile = open(outname, 'w') - print 'writing file "%s"' % outname - print >>outfile, self.header + print('writing file "%s"' % outname) + outfile.write(self.header + '\n') set = self.sets[set_name] entities = [(e.lower(), e) for e in set.keys()] entities.sort() @@ -193,9 +196,9 @@ class CharacterEntitySetExtractor: if int(code, 16) > 0xFFFF: return 1 # wide-Unicode character codes = ' '.join(['U+%s' % code for code in charid[1:].split('-')]) - print >>outfile, ('.. %-*s unicode:: %s .. %s' - % (longest + 2, '|' + entity_name + '|', - codes, self.descriptions[charid])) + outfile.write('.. %-*s unicode:: %s .. %s\n' + % (longest + 2, '|' + entity_name + '|', + codes, self.descriptions[charid])) if __name__ == '__main__': diff --git a/tools/quicktest.py b/tools/quicktest.py index bb3a8aa7a..70df51ca4 100755 --- a/tools/quicktest.py +++ b/tools/quicktest.py @@ -52,21 +52,21 @@ options = [('pretty', 'p', the data structure: (long option, short option, description).""" def usage(): - print usage_header + print(usage_header) for longopt, shortopt, description in options: if longopt[-1:] == '=': opts = '-%s arg, --%sarg' % (shortopt, longopt) else: opts = '-%s, --%s' % (shortopt, longopt) - print '%-15s' % opts, + sys.stdout.write('%-15s' % opts) if len(opts) > 14: - print '%-16s' % '\n', + sys.stdout.write('%-16s' % '\n') while len(description) > 60: limit = description.rindex(' ', 0, 60) - print description[:limit].strip() + print(description[:limit].strip()) description = description[limit + 1:] - print '%-15s' % ' ', - print description + sys.stdout.write('%-15s' % ' ') + print(description) def _pretty(input, document, optargs): return document.pformat() @@ -142,9 +142,9 @@ def posixGetArgs(argv): usage() sys.exit() elif o in ['-V', '--version']: - print >>sys.stderr, ('quicktest.py (Docutils %s [%s])' - % (docutils.__version__, - docutils.__version_details__)) + sys.stderr.write('quicktest.py (Docutils %s [%s])\n' % + (docutils.__version__, + docutils.__version_details__)) sys.exit() elif o in ['-r', '--rawxml']: outputFormat = 'rawxml' @@ -162,9 +162,9 @@ def posixGetArgs(argv): elif o in ['-d', '--debug']: optargs['debug'] = 1 else: - raise getopt.GetoptError, "getopt should have saved us!" + raise getopt.GetoptError("getopt should have saved us!") if len(args) > 2: - print 'Maximum 2 arguments allowed.' + print('Maximum 2 arguments allowed.') usage() sys.exit(1) inputFile = sys.stdin diff --git a/tools/test/test_buildhtml.py b/tools/test/test_buildhtml.py index 4c9f38978..53baf6559 100644 --- a/tools/test/test_buildhtml.py +++ b/tools/test/test_buildhtml.py @@ -25,18 +25,31 @@ Build-HTML Options import unittest import os import re +try: + import tempfile +except ImportError: + pass +try: + from subprocess import Popen, PIPE, STDOUT +except ImportError: + pass def process_and_return_filelist(options): dirs = [] files = [] - cin, cout = os.popen4("../buildhtml.py "+options) + try: + p = Popen("../buildhtml.py "+options, shell=True, + stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) + (cin, cout) = (p.stdin, p.stdout) + except NameError: + cin, cout = os.popen4("../buildhtml.py "+options) while 1: - ln = cout.readline() - if not ln: + line = cout.readline() + if not line: break # BUG no colon in filename/path allowed - item = ln.split(":")[-1].strip() - if ln.startswith(" "): + item = line.split(":")[-1].strip() + if line.startswith(" "): files.append(item) else: dirs.append(item) @@ -58,8 +71,12 @@ class BuildHtmlTests(unittest.TestCase): ) def setUp(self): - self.root = os.tempnam() - os.mkdir(self.root) + try: + self.root = tempfile.mkdtemp() + except NameError: + self.root = os.tempnam() + os.mkdir(self.root) + for s in self.tree: s = os.path.join(self.root, s) if not "." in s: -- 2.11.4.GIT