From b5ddb0b0204cd907683573356eb7965516c005da Mon Sep 17 00:00:00 2001 From: milde Date: Thu, 7 Jun 2007 11:26:14 +0000 Subject: [PATCH] moved the DocutilsInterface class to pygments_code_block_directive.py git-svn-id: http://svn.berlios.de/svnroot/repos/pylit/trunk@57 fb71aa59-6827-0410-b536-ee2229a4f8e3 --- rstdocs/features/pygments_code_block_directive.py | 40 +++++++- rstdocs/features/pygments_docutils_interface.py | 117 +++++++--------------- 2 files changed, 75 insertions(+), 82 deletions(-) rewrite rstdocs/features/pygments_docutils_interface.py (94%) diff --git a/rstdocs/features/pygments_code_block_directive.py b/rstdocs/features/pygments_code_block_directive.py index fa8708e..e0d5267 100644 --- a/rstdocs/features/pygments_code_block_directive.py +++ b/rstdocs/features/pygments_code_block_directive.py @@ -14,6 +14,8 @@ # 2007-06-05 Separate docutils formatter script # Use pygments' CSS class names (like the html formatter) # allowing the use of pygments-produced style sheets. +# 2007-06-07 Re-include the formatting of the parsed tokens +# (class DocutilsInterface) # ========== =========================================================== # # :: @@ -29,7 +31,7 @@ from docutils import nodes from docutils.parsers.rst import directives import pygments from pygments.lexers import get_lexer_by_name -from pygments_docutils_formatter import DocutilsInterface +from pygments.formatters.html import _get_ttype_class # Customisation # ------------- @@ -39,6 +41,42 @@ from pygments_docutils_formatter import DocutilsInterface unstyled_tokens = [''] +# DocutilsInterface +# ----------------- + +# This interface class combines code from +# pygments.formatters.html and pygments.formatters.others:: + +class DocutilsInterface(object): + """Yield tokens for addition to the docutils document tree. + + Merge subsequent tokens of the same token-type. + + Yields the tokens as ``(ttype_class, value)`` tuples, + where ttype_class is taken from pygments.token.STANDARD_TYPES and + corresponds to the class argument used in pygments html output. + + """ + name = 'docutils' + # aliases = ['docutils tokens'] + + def __init__(self, tokensource): + self.tokensource = tokensource + + def __iter__(self): + lasttype = None + lastval = u'' + for ttype, value in self.tokensource: + if ttype is lasttype: + lastval += value + else: + if lasttype: + yield(_get_ttype_class(lasttype), lastval) + lastval = value + lasttype = ttype + yield(_get_ttype_class(lasttype), lastval) + + # code_block_directive # -------------------- diff --git a/rstdocs/features/pygments_docutils_interface.py b/rstdocs/features/pygments_docutils_interface.py dissimilarity index 94% index 5c30bfd..756a4c1 100755 --- a/rstdocs/features/pygments_docutils_interface.py +++ b/rstdocs/features/pygments_docutils_interface.py @@ -1,81 +1,36 @@ -import pygments -import pygments.lexers -from pygments.formatter import Formatter -from pygments.formatters.html import _get_ttype_class - - -# This formatter class combines code from -# pygments.formatters.html and pygments.formatters.others:: - -class DocutilsInterface(object): - """Yield tokens for addition to the docutils document tree. - - Merge subsequent tokens of the same token-type. - Does not write to a file but yields the tokens as - ``(ttype_class, value)`` tuples. - - Where ttype_class is taken from pygments.token.STANDARD_TYPES) and - corresponds to the class argument used in pygments html output. - - This formatter differs from the "normal" pygments formatters as it is - solely intended for programmatic use. It - The docutils 'code_block' directive will use this to convert the parsed - tokens to a doctree element with nodes for tokes - with ttype_class != ''. - - """ - name = 'docutils' - # aliases = ['docutils tokens'] - - def __init__(self, tokensource, **options): - self.tokensource = tokensource - - def __iter__(self): - lasttype = None - lastval = u'' - for ttype, value in self.tokensource: - if ttype is lasttype: - lastval += value - else: - if lasttype: - yield(_get_ttype_class(lasttype), lastval) - lastval = value - lasttype = ttype - yield(_get_ttype_class(lasttype), lastval) - - -# Test the parsing and formatting by pygments: - -if __name__ == "__main__": - - from docutils import nodes, utils, core - - source_string = """\ -def my_function(): - "just a test" - print 8/2 -""" - - lexer = pygments.lexers.get_lexer_by_name('python') - tokens = list(pygments.lex(source_string, lexer)) - document = utils.new_document('generated') - literal_block = nodes.literal_block(raw_code=source_string.splitlines(True), - classes=["code-block", "python"]) - document += literal_block - - # You could add e.g. 'p' (Token.Punctuation) to unstyled_tokens. - unstyled_tokens = ['', ] - for cls, value in DocutilsInterface(tokens): - if cls in unstyled_tokens: - # insert as Text to decrease the verbosity of the output. - node = nodes.Text(value, value) - else: - node = nodes.inline(value, value, classes=[cls]) - literal_block += node - - # print core.publish_from_doctree(document, writer_name='html') - # print core.publish_from_doctree(document, writer_name='pseudoxml') - print core.publish_from_doctree(document, writer_name='xml') - # print core.publish_from_doctree(document, writer_name='latex') - # print core.publish_from_doctree(document, writer_name='newlatex2e') - +# Test the parsing and formatting by pygments: + + +from docutils import nodes, utils, core +import pygments.lexers +from pygments_code_block_directive import DocutilsInterface + +source_string = """\ +def my_function(): + "just a test" + print 8/2 +""" + +lexer = pygments.lexers.get_lexer_by_name('python') +tokens = list(pygments.lex(source_string, lexer)) +document = utils.new_document('generated') +literal_block = nodes.literal_block(raw_code=source_string.splitlines(True), + classes=["code-block", "python"]) +document += literal_block + +# You could add e.g. 'p' (Token.Punctuation) to unstyled_tokens. +unstyled_tokens = ['', ] +for cls, value in DocutilsInterface(tokens): + if cls in unstyled_tokens: + # insert as Text to decrease the verbosity of the output. + node = nodes.Text(value, value) + else: + node = nodes.inline(value, value, classes=[cls]) + literal_block += node + +writer_names = ('html', 'pseudoxml', 'xml', 'latex', 'newlatex2e', 's5') +for name in writer_names[2:3]: + print "\nusing writer %r\n" % name + print core.publish_from_doctree(document, writer_name=name) + + -- 2.11.4.GIT