rstdocs/features/pygments_docutils_formatter.py

   1 import pygments
   2 import pygments.lexers
   3 from pygments.formatter import Formatter
   4 from pygments.formatters.html import _get_ttype_class
   5
   6
   7 # This formatter class combines code from
   8 # pygments.formatters.html and pygments.formatters.others::
   9
  10 class DocutilsInterface(object):
  11     """Yield tokens for addition to the docutils document tree.
  12
  13     Merge subsequent tokens of the same token-type.
  14     Does not write to a file but yields the tokens as
  15     ``(ttype_class, value)`` tuples.
  16
  17     Where ttype_class is taken from pygments.token.STANDARD_TYPES) and
  18     corresponds to the class argument used in pygments html output.
  19
  20     This formatter differs from the "normal" pygments formatters as it is
  21     solely intended for programmatic use. It
  22     The docutils 'code_block' directive will use this to convert the parsed
  23     tokens to a <literal_block> doctree element with <inline> nodes for tokes
  24     with ttype_class != ''.
  25
  26     """
  27     name = 'docutils'
  28     # aliases = ['docutils tokens']
  29
  30     def __init__(self, tokensource, **options):
  31         self.tokensource = tokensource
  32
  33     def __iter__(self):
  34         lasttype = None
  35         lastval = u''
  36         for ttype, value in self.tokensource:
  37             if ttype is lasttype:
  38                 lastval += value
  39             else:
  40                 if lasttype:
  41                     yield(_get_ttype_class(lasttype), lastval)
  42                 lastval = value
  43                 lasttype = ttype
  44         yield(_get_ttype_class(lasttype), lastval)
  45
  46
  47 # Test the parsing and formatting by pygments:
  48
  49 if __name__ == "__main__":
  50
  51     from docutils import nodes, utils, core
  52
  53     source_string = """\
  54 def my_function():
  55     "just a test"
  56     print 8/2
  57 """
  58
  59     lexer = pygments.lexers.get_lexer_by_name('python')
  60     tokens = list(pygments.lex(source_string, lexer))
  61     document = utils.new_document('generated')
  62     literal_block = nodes.literal_block(raw_code=source_string.splitlines(True),
  63                                         classes=["code-block", "python"])
  64     document += literal_block
  65
  66     # You could add e.g. 'p' (Token.Punctuation) to unstyled_tokens.
  67     unstyled_tokens = ['', ]
  68     for cls, value in DocutilsInterface(tokens):
  69         if cls in unstyled_tokens:
  70             # insert as Text to decrease the verbosity of the output.
  71             node = nodes.Text(value, value)
  72         else:
  73             node = nodes.inline(value, value, classes=[cls])
  74         literal_block += node
  75
  76     # print core.publish_from_doctree(document, writer_name='html')
  77     # print core.publish_from_doctree(document, writer_name='pseudoxml')
  78     print core.publish_from_doctree(document, writer_name='xml')
  79     # print core.publish_from_doctree(document, writer_name='latex')
  80     # print core.publish_from_doctree(document, writer_name='newlatex2e')
  81