rstdocs/features/pygments_docutils_formatter.py

   1 import pygments
   2 import pygments.lexers
   3 from pygments.formatter import Formatter
   4 from pygments.formatters.html import _get_ttype_class
   5
   6
   7 # This formatter class combines code from
   8 # pygments.formatters.html and pygments.formatters.others::
   9
  10 class DocutilsFormatter(Formatter):
  11     """Yield tokens for addition to the docutils document tree.
  12
  13     Merge subsequent tokens of the same token-type.
  14     Does not write to a file but yields the tokens as
  15     ``(ttype_class, value)`` tuples.
  16
  17     Where ttype_class is taken from pygments.token.STANDARD_TYPES) and
  18     corresponds to the class argument used in pygments html output.
  19
  20     This formatter differs from the "normal" pygments formatters as it is
  21     solely intended for programmatic use. It
  22     The docutils 'code_block' directive will use this to convert the parsed
  23     tokens to a <literal_block> doctree element with <inline> nodes for tokes
  24     with ttype_class != ''.
  25
  26     """
  27     name = 'docutils'
  28     # aliases = ['docutils tokens']
  29
  30     def __init__(self, tokensource, **options):
  31         Formatter.__init__(self, **options)
  32         self.tokensource = tokensource
  33
  34     def __iter__(self):
  35         lasttype = None
  36         lastval = u''
  37         for ttype, value in self.tokensource:
  38             if ttype is lasttype:
  39                 lastval += value
  40             else:
  41                 if lasttype:
  42                     yield(_get_ttype_class(lasttype), lastval)
  43                 lastval = value
  44                 lasttype = ttype
  45         yield(_get_ttype_class(lasttype), lastval)
  46
  47
  48 # Test the parsing and formatting by pygments:
  49
  50 if __name__ == "__main__":
  51
  52     from docutils import nodes, utils, core
  53
  54     source_string = """\
  55 def my_function():
  56     "just a test"
  57     print 8/2
  58 """
  59
  60     lexer = pygments.lexers.get_lexer_by_name('python')
  61     tokens = list(pygments.lex(source_string, lexer))
  62     document = utils.new_document('generated')
  63     literal_block = nodes.literal_block(raw_code=source_string.splitlines(True),
  64                                         classes=["code-block", "python"])
  65     document += literal_block
  66
  67     # You could add e.g. 'p' (Token.Punctuation) to unstyled_tokens.
  68     unstyled_tokens = ['', ]
  69     for cls, value in DocutilsFormatter(tokens):
  70         if cls in unstyled_tokens:
  71             # insert as Text to decrease the verbosity of the output.
  72             node = nodes.Text(value, value)
  73         else:
  74             node = nodes.inline(value, value, classes=[cls])
  75         literal_block += node
  76
  77     # print core.publish_from_doctree(document, writer_name='html')
  78     # print core.publish_from_doctree(document, writer_name='pseudoxml')
  79     print core.publish_from_doctree(document, writer_name='xml')
  80     # print core.publish_from_doctree(document, writer_name='latex')
  81     # print core.publish_from_doctree(document, writer_name='newlatex2e')
  82