From 18669ca369062c90e5e2dce6147c2d5fa5316ccf Mon Sep 17 00:00:00 2001 From: milde Date: Tue, 5 Jun 2007 07:54:21 +0000 Subject: [PATCH] separate DocutilsFormatter, updated proposal for "code-block" directive git-svn-id: http://svn.berlios.de/svnroot/repos/pylit/trunk@53 fb71aa59-6827-0410-b536-ee2229a4f8e3 --- rstdocs/features/code-block.css | 119 ++++++++++++++++++++++ rstdocs/features/myfunction.py.html | 29 ++++++ rstdocs/features/myfunction.py.tex | 84 +++++++++++++++ rstdocs/features/myfunction.py.txt | 11 ++ rstdocs/features/myfunction.py.xml | 10 ++ rstdocs/features/pygments_code_block_directive.py | 108 ++++++++++++++++++++ rstdocs/features/pygments_docutils_formatter.py | 82 +++++++++++++++ rstdocs/features/rst2html-highlight | 36 +++++++ rstdocs/features/syntax-highlight.txt | 84 +++++++++------ 9 files changed, 534 insertions(+), 29 deletions(-) create mode 100644 rstdocs/features/code-block.css create mode 100644 rstdocs/features/myfunction.py.html create mode 100644 rstdocs/features/myfunction.py.tex create mode 100644 rstdocs/features/myfunction.py.txt create mode 100644 rstdocs/features/myfunction.py.xml create mode 100644 rstdocs/features/pygments_code_block_directive.py create mode 100755 rstdocs/features/pygments_docutils_formatter.py create mode 100755 rstdocs/features/rst2html-highlight diff --git a/rstdocs/features/code-block.css b/rstdocs/features/code-block.css new file mode 100644 index 0000000..f05d26d --- /dev/null +++ b/rstdocs/features/code-block.css @@ -0,0 +1,119 @@ +/* Stylesheet for pygments enhanced reStructured Text */ +/* ================================================== */ + +/* :Author: Guenter Milde */ +/* :Copyright: 2007 G. Milde */ +/* This stylesheet is released under the GPL v. 2 or later */ + +/* This stylesheet provides syntax highlight for documents generated with a */ +/* pygments_ enhanced reStructured Text -> html converter. */ + +/* Import the default docutils style sheet */ +/* --------------------------------------- */ +/* :: */ + +@import url("/stylesheets/html4css1.css"); + +/* Indent the code block */ +/* --------------------- */ + +/* Content copied from the `html4css1.css` rule for literal blocks. */ +/* Selector adapted to the output of Pygments_. :: */ + +div.highlight { + margin-left: 2em ; + margin-right: 2em ; + background-color: #eeeeee + } + + +/* Layout for code block tokens */ +/* ---------------------------- */ + +.comment { color: #008800; font-style: italic } +.error { border: 1px solid #FF0000 } +.keyword { color: #AA22FF; font-weight: bold } +.operator { color: #666666 } +.comment.multiline { color: #008800; font-style: italic } +.comment.preproc { color: #008800 } +.comment.single { color: #008800; font-style: italic } +.generic.deleted { color: #A00000 } +.generic.emph { font-style: italic } +.generic.error { color: #FF0000 } +.generic.heading { color: #000080; font-weight: bold } +.generic.inserted { color: #00A000 } +.generic.output { color: #808080 } +.generic.prompt { color: #000080; font-weight: bold } +.generic.strong { font-weight: bold } +.generic.subheading { color: #800080; font-weight: bold } +.generic.traceback { color: #0040D0 } +.keyword.constant { color: #AA22FF; font-weight: bold } +.keyword.declaration { color: #AA22FF; font-weight: bold } +.keyword.pseudo { color: #AA22FF } +.keyword.reserved { color: #AA22FF; font-weight: bold } +.keyword.type { color: #AA22FF; font-weight: bold } +.literal.number { color: #666666 } +.literal.string { color: #BB4444 } +.name.attribute { color: #BB4444 } +.name.builtin { color: #AA22FF } +.name.class { color: #0000FF } +.name.constant { color: #880000 } +.name.decorator { color: #AA22FF } +.name.entity { color: #999999; font-weight: bold } +.name.exception { color: #D2413A; font-weight: bold } +.name.function { color: #00A000 } +.name.label { color: #A0A000 } +.name.namespace { color: #0000FF; font-weight: bold } +.name.tag { color: #008000; font-weight: bold } +.name.variable { color: #B8860B } +.operator.word { color: #AA22FF; font-weight: bold } +.literal.number.float { color: #666666 } +.literal.number.hex { color: #666666 } +.literal.number.integer { color: #666666 } +.literal.number.oct { color: #666666 } +.literal.string.backtick { color: #BB4444 } +.literal.string.char { color: #BB4444 } +.literal.string.doc { color: #BB4444; font-style: italic } +.literal.string.double { color: #BB4444 } +.literal.string.escape { color: #BB6622; font-weight: bold } +.literal.string.heredoc { color: #BB4444 } +.literal.string.interpol { color: #BB6688; font-weight: bold } +.literal.string.other { color: #008000 } +.literal.string.regex { color: #BB6688 } +.literal.string.single { color: #BB4444 } +.literal.string.symbol { color: #B8860B } +.name.builtin.pseudo { color: #AA22FF } +.name.variable.class { color: #B8860B } +.name.variable.global { color: #B8860B } +.name.variable.instance { color: #B8860B } +.literal.number.integer.long { color: #666666 } + +/* .. _pygments: http://pygments.org/ */ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rstdocs/features/myfunction.py.html b/rstdocs/features/myfunction.py.html new file mode 100644 index 0000000..29aaddd --- /dev/null +++ b/rstdocs/features/myfunction.py.html @@ -0,0 +1,29 @@ + + + + + + + + + + +
+

This is a test of the new code-block directive:

+ +
+def my_function():
+    "just a test"
+    print 8/2
+
+
+
+
+
+Generated on: 2007-06-05. + +
+ + diff --git a/rstdocs/features/myfunction.py.tex b/rstdocs/features/myfunction.py.tex new file mode 100644 index 0000000..d8c934c --- /dev/null +++ b/rstdocs/features/myfunction.py.tex @@ -0,0 +1,84 @@ +\documentclass[10pt,a4paper,english]{article} +\usepackage{babel} +\usepackage{ae} +\usepackage{aeguill} +\usepackage{shortvrb} +\usepackage[latin1]{inputenc} +\usepackage{tabularx} +\usepackage{longtable} +\setlength{\extrarowheight}{2pt} +\usepackage{amsmath} +\usepackage{graphicx} +\usepackage{color} +\usepackage{multirow} +\usepackage{ifthen} +\usepackage[colorlinks=true,linkcolor=blue,urlcolor=blue]{hyperref} +\usepackage[DIV12]{typearea} +%% generator Docutils: http://docutils.sourceforge.net/ +\newlength{\admonitionwidth} +\setlength{\admonitionwidth}{0.9\textwidth} +\newlength{\docinfowidth} +\setlength{\docinfowidth}{0.9\textwidth} +\newlength{\locallinewidth} +\newcommand{\optionlistlabel}[1]{\bf #1 \hfill} +\newenvironment{optionlist}[1] +{\begin{list}{} + {\setlength{\labelwidth}{#1} + \setlength{\rightmargin}{1cm} + \setlength{\leftmargin}{\rightmargin} + \addtolength{\leftmargin}{\labelwidth} + \addtolength{\leftmargin}{\labelsep} + \renewcommand{\makelabel}{\optionlistlabel}} +}{\end{list}} +\newlength{\lineblockindentation} +\setlength{\lineblockindentation}{2.5em} +\newenvironment{lineblock}[1] +{\begin{list}{} + {\setlength{\partopsep}{\parskip} + \addtolength{\partopsep}{\baselineskip} + \topsep0pt\itemsep0.15\baselineskip\parsep0pt + \leftmargin#1} + \raggedright} +{\end{list}} +% begin: floats for footnotes tweaking. +\setlength{\floatsep}{0.5em} +\setlength{\textfloatsep}{\fill} +\addtolength{\textfloatsep}{3em} +\renewcommand{\textfraction}{0.5} +\renewcommand{\topfraction}{0.5} +\renewcommand{\bottomfraction}{0.5} +\setcounter{totalnumber}{50} +\setcounter{topnumber}{50} +\setcounter{bottomnumber}{50} +% end floats for footnotes +% some commands, that could be overwritten in the style file. +\newcommand{\rubric}[1]{\subsection*{~\hfill {\it #1} \hfill ~}} +\newcommand{\titlereference}[1]{\textsl{#1}} +% end of "some commands" +\title{} +\author{} +\date{} +\raggedbottom +\begin{document} + +\setlength{\locallinewidth}{\linewidth} + +This is a test of the new code-block directive: +% Translate this document to HTML with a pygments enhanced frontend, e.g. +% +% rst2html-highlight --stylesheet=pygments-default.css --link-stylesheet +\begin{quote}{\ttfamily \raggedright \noindent +\docutilsroleNone{def}~\docutilsroleNone{my{\_}function}\docutilsroleNone{():}~\\ +~~~~\docutilsroleNone{"just~a~test"}~\\ +~~~~\docutilsroleNone{print}~\docutilsroleNone{8}\docutilsroleNone{/}\docutilsroleNone{2}~\\ + +}\end{quote} + +\begin{center}\small + +Generated on: 2007-06-05. + + +\end{center} + +\end{document} diff --git a/rstdocs/features/myfunction.py.txt b/rstdocs/features/myfunction.py.txt new file mode 100644 index 0000000..926733d --- /dev/null +++ b/rstdocs/features/myfunction.py.txt @@ -0,0 +1,11 @@ +This is a test of the new code-block directive: + +.. Translate this document to HTML with a pygments enhanced frontend, e.g. + + rst2html-highlight --stylesheet=pygments-default.css --link-stylesheet + +.. code-block:: python + + def my_function(): + "just a test" + print 8/2 diff --git a/rstdocs/features/myfunction.py.xml b/rstdocs/features/myfunction.py.xml new file mode 100644 index 0000000..564e57e --- /dev/null +++ b/rstdocs/features/myfunction.py.xml @@ -0,0 +1,10 @@ + + + +
Generated on: 2007-06-05. +
This is a test of the new code-block directive:Translate this document to HTML with a pygments enhanced frontend, e.g. + +rst2html-highlight --stylesheet=pygments-default.css --link-stylesheetdef my_function(): + "just a test" + print 8/2 + diff --git a/rstdocs/features/pygments_code_block_directive.py b/rstdocs/features/pygments_code_block_directive.py new file mode 100644 index 0000000..61918d8 --- /dev/null +++ b/rstdocs/features/pygments_code_block_directive.py @@ -0,0 +1,108 @@ +#!/usr/bin/python + +# :Author: a Pygments author|contributor; Felix Wiemann; Guenter Milde +# :Date: $Date: $ +# :Copyright: This module has been placed in the public domain. +# +# This is a merge of `Using Pygments in ReST documents`_ from the pygments_ +# documentation, and a `proof of concept`_ by Felix Wiemann. +# +# ========== =========================================================== +# 2007-06-01 Removed redundancy from class values. +# 2007-06-04 Merge of successive tokens of same type +# (code taken from pygments.formatters.others). +# 2007-06-05 Separate docutils formatter script +# Use pygments' CSS class names (like the html formatter) +# allowing the use of pygments-produced style sheets. +# ========== =========================================================== +# +# :: + +"""Define and register a code-block directive using pygments +""" + +# Requirements +# ------------ +# :: + +from docutils import nodes +from docutils.parsers.rst import directives +import pygments +from pygments.lexers import get_lexer_by_name +from pygments_docutils_formatter import DocutilsFormatter + +# Customisation +# ------------- +# +# Do not insert inline nodes for the following tokens. +# (You could add e.g. Token.Punctuation like ``['', 'p']``.) :: + +unstyled_tokens = [''] + + +# code_block_directive +# -------------------- +# :: + +def code_block_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + language = arguments[0] + # create a literal block element and set class argument + code_block = nodes.literal_block(raw_content=content, + classes=["code-block", language]) + # Get lexer for language (use text as fallback) + try: + lexer = get_lexer_by_name(language) + except ValueError: + lexer = get_lexer_by_name('text') + + # parse content with pygments + tokens = list(pygments.lex(u'\n'.join(content), lexer)) + + for cls, value in DocutilsFormatter(tokens): + if cls in unstyled_tokens: + # insert as Text to decrease the verbosity of the output. + code_block += nodes.Text(value, value) + else: + code_block += nodes.inline(value, value, classes=[cls]) + + return [code_block] + + +# Register Directive +# ------------------ +# :: + +code_block_directive.arguments = (1, 0, 1) +code_block_directive.content = 1 +directives.register_directive('code-block', code_block_directive) + +# .. _doctutile: http://docutils.sf.net/ +# .. _pygments: http://pygments.org/ +# .. _Using Pygments in ReST documents: http://pygments.org/docs/rstdirective/ +# .. _proof of concept: +# http://article.gmane.org/gmane.text.docutils.user/3689 +# +# Test output +# ----------- +# +# If called from the command line, call the docutils publisher to render the +# input:: + +if __name__ == '__main__': + from docutils.core import publish_cmdline, default_description + description = "code-block directive test output" + default_description + try: + import locale + locale.setlocale(locale.LC_ALL, '') + except: + pass + # Uncomment the desired output format: + publish_cmdline(writer_name='pseudoxml', description=description) + # publish_cmdline(writer_name='xml', description=description) + # publish_cmdline(writer_name='html', description=description) + # publish_cmdline(writer_name='latex', description=description) + # publish_cmdline(writer_name='newlatex2e', description=description) + + + diff --git a/rstdocs/features/pygments_docutils_formatter.py b/rstdocs/features/pygments_docutils_formatter.py new file mode 100755 index 0000000..60ffa7d --- /dev/null +++ b/rstdocs/features/pygments_docutils_formatter.py @@ -0,0 +1,82 @@ +import pygments +import pygments.lexers +from pygments.formatter import Formatter +from pygments.formatters.html import _get_ttype_class + + +# This formatter class combines code from +# pygments.formatters.html and pygments.formatters.others:: + +class DocutilsFormatter(Formatter): + """Yield tokens for addition to the docutils document tree. + + Merge subsequent tokens of the same token-type. + Does not write to a file but yields the tokens as + ``(ttype_class, value)`` tuples. + + Where ttype_class is taken from pygments.token.STANDARD_TYPES) and + corresponds to the class argument used in pygments html output. + + This formatter differs from the "normal" pygments formatters as it is + solely intended for programmatic use. It + The docutils 'code_block' directive will use this to convert the parsed + tokens to a doctree element with nodes for tokes + with ttype_class != ''. + + """ + name = 'docutils' + # aliases = ['docutils tokens'] + + def __init__(self, tokensource, **options): + Formatter.__init__(self, **options) + self.tokensource = tokensource + + def __iter__(self): + lasttype = None + lastval = u'' + for ttype, value in self.tokensource: + if ttype is lasttype: + lastval += value + else: + if lasttype: + yield(_get_ttype_class(lasttype), lastval) + lastval = value + lasttype = ttype + yield(_get_ttype_class(lasttype), lastval) + + +# Test the parsing and formatting by pygments: + +if __name__ == "__main__": + + from docutils import nodes, utils, core + + source_string = """\ +def my_function(): + "just a test" + print 8/2 +""" + + lexer = pygments.lexers.get_lexer_by_name('python') + tokens = list(pygments.lex(source_string, lexer)) + document = utils.new_document('generated') + literal_block = nodes.literal_block(raw_code=source_string.splitlines(True), + classes=["code-block", "python"]) + document += literal_block + + # You could add e.g. 'p' (Token.Punctuation) to unstyled_tokens. + unstyled_tokens = ['', ] + for cls, value in DocutilsFormatter(tokens): + if cls in unstyled_tokens: + # insert as Text to decrease the verbosity of the output. + node = nodes.Text(value, value) + else: + node = nodes.inline(value, value, classes=[cls]) + literal_block += node + + # print core.publish_from_doctree(document, writer_name='html') + # print core.publish_from_doctree(document, writer_name='pseudoxml') + print core.publish_from_doctree(document, writer_name='xml') + # print core.publish_from_doctree(document, writer_name='latex') + # print core.publish_from_doctree(document, writer_name='newlatex2e') + diff --git a/rstdocs/features/rst2html-highlight b/rstdocs/features/rst2html-highlight new file mode 100755 index 0000000..737a8c8 --- /dev/null +++ b/rstdocs/features/rst2html-highlight @@ -0,0 +1,36 @@ +#!/usr/bin/python + +# :Author: David Goodger, a Pygments author|contributor, Guenter Milde +# :Date: $Date: $ +# :Copyright: This module has been placed in the public domain. + +# This is a merge of the docutils_ `rst2html` front end with an extension +# suggestion taken from the pygments_ documentation. + +""" +A front end to docutils, producing HTML with syntax colouring using pygments + +Generates (X)HTML documents from standalone reStructuredText sources. Uses +`pygments` to parse and mark up the content of ``.. code-block::` directives. +Needs an adapted stylesheet +""" + +try: + import locale + locale.setlocale(locale.LC_ALL, '') +except: + pass + +from docutils.core import publish_cmdline, default_description + +# Define and register a new directive `code-block` that uses the `pygments`_ +# source highlighter to render code in color. +import pygments_code_block_directive + +# Call the docutils publisher to render the input as html:: +description = __doc__ + default_description +publish_cmdline(writer_name='html', description=description) + +# .. _doctutile: http://docutils.sf.net/ +# .. _pygments: http://pygments.org/ +# .. _Using Pygments in ReST documents: http://pygments.org/docs/rstdirective/ diff --git a/rstdocs/features/syntax-highlight.txt b/rstdocs/features/syntax-highlight.txt index 6feaa99..1ad3b75 100644 --- a/rstdocs/features/syntax-highlight.txt +++ b/rstdocs/features/syntax-highlight.txt @@ -11,15 +11,17 @@ file: for-else-test.py.html file: pygments-default.css file: pygments-default.sty - file: code-block.css + file: pygments_docutils_formatter.py + file: pygments_code_block_directive.py + file: pygments_code_block_directive.py.html + file: rst2html-highlight file: myfunction.py.txt file: myfunction.py.xml + file: myfunction.py.pseudoxml file: myfunction.py.html file: myfunction.py.tex - file: pygments_code_block_directive.py - file: rst2html-code-block + file: myfunction.py.newlatex2.tex file: literal-blocks.pdf - file: pygments_with_docutils.py /restindex Syntax Highlight @@ -210,8 +212,8 @@ Stylesheets: Proposal for a code-block directive in docutils ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In a `post to the docutils users list`__, David Goodger wrote (after an all too -long discussion): +In a `post to the docutils users list`__, David Goodger wrote (after an all +too long discussion): Here are my pronouncements: @@ -228,44 +230,68 @@ long discussion): __ http://article.gmane.org/gmane.text.docutils.user/3923 -Felix Wiemann provided a `proof of concept script`_ that utilizes the -pygments_ parser to do this. In contrast to the `pygments enhanced docutils -front-ends`_, this script produces a doctree element -with nodes instead of element, i.e. it is independent of the -output format. -A modified version, `pygments_with_docutils.py`_, parses the small example -file `myfunction.py.txt`_ into a "rich" doctree (`myfunction.py.xml`_). +Parsing +""""""" + +Felix Wiemann provided a `proof of concept`_ script that utilizes the +pygments_ parser to parse a source code string and store the result in a +"rich" element with "classified" nodes. + +A modified version, `pygments_code_block_directive`_, (source: +`pygments_code_block_directive.py`_), defines and registers a "code-block" +directive that parses its content, stores it in a . It uses a +DocutilsFormatter (`pygments_docutils_formatter.py`_) to classify the tokens +using short CSS class names identical to pygments HTML output. + +The XML rendering of the small example file `myfunction.py.txt`_ looks like +`myfunction.py.xml`_. + +Writing +""""""" The writers can use the class information in the elements to render -the tokens accordingly. +the tokens. They should ignore the class information if they are unable to +use it or to pass it on. -* The HTML writer only needs an adapted CSS style sheet. - - * The rst2html-code-block_ front end registers the "code-block" directive and +HTML + The "html" writer works out of the box. + + * The rst2html-highlight_ front end registers the "code-block" directive and converts an input file to html. - - * The `code-block.css`_ stylesheet provides the layout information. - + * Styling is done with the adapted CSS style sheet `pygments-default.css`_ + based on docutils' default stylesheet and the output of + ``pygmentize -S default -f html``. * The result looks like `myfunction.py.html`_. -* The LaTeX writer currently fails to handle the "classified" - doctree elements. - - The result looks like `myfunction.py.tex`_. +LaTeX + Latex writers must be updated to handle the "rich" element + correct. + + * The "latex" writer currently fails to handle the "classified" + doctree elements. The output `myfunction.py.tex`_ contains undefined + control sequences ``\docutilsroleNone``. + + * The "newlatex2e" writer produces a valid LaTeX document + (`myfunction.py.newlatex2e.tex`_). However the `pdflatex` output looks + mixed up a bit (`myfunction.py.newlatex2e.pdf`_). The pygments-produced + style file will not work with "newlatex2e" output. -.. _proof of concept script: +.. _proof of concept: http://article.gmane.org/gmane.text.docutils.user/3689 - -.. _pygments_with_docutils.py: pygments_with_docutils.py +.. _pygments_code_block_directive.py: pygments_code_block_directive.py +.. _pygments_code_block_directive: pygments_code_block_directive.py.html .. _myfunction.py.txt: myfunction.py.txt .. _myfunction.py.xml: myfunction.py.xml .. _myfunction.py.html: myfunction.py.html .. _myfunction.py.tex: myfunction.py.tex -.. _rst2html-code-block: rst2html-code-block +.. _myfunction.py.newlatex2e.tex: myfunction.py.newlatex2e.tex +.. _myfunction.py.newlatex2e.pdf: myfunction.py.newlatex2e.pdf +.. _rst2html-highlight: rst2html-highlight .. _code-block.css: code-block.css + Configurable literal block directive ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -487,5 +513,5 @@ Some considerations about the implementation: .. _pygments-default.css.txt: pygments-default.css.txt .. _pygments-default.css.html: pygments-default.css.html .. _pygments-default.sty: pygments-default.sty -.. _pygments_with_docutils.py: pygments_with_docutils.py +.. _pygments_code_block_directive.py: pygments_code_block_directive.py -- 2.11.4.GIT