From ad40b58ebfc2ff17ce8802ea40b9fb79979b3940 Mon Sep 17 00:00:00 2001 From: milde Date: Tue, 1 Dec 2020 11:38:25 +0000 Subject: [PATCH] Add support for Markdown. Provide a wrapper for the 3rd party `recommonmark`__ Markdown parser already in use in Sphinx. Add test cases for CommonMark parsing. Also: Correct copyright note and set executable bit on two tests. Fix small typo in test content. __ https://pypi.org/project/recommonmark/ git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk@8585 929543f6-e4f2-0310-98a6-ba3bd3dd1d04 --- docutils/docutils/parsers/__init__.py | 23 +- docutils/docutils/parsers/recommonmark_wrapper.py | 118 +++++ docutils/test/DocutilsTestSupport.py | 38 +- .../test_parsers/test_recommonmark/__init__.py | 15 + .../test_recommonmark/test_block_quotes.py | 110 +++++ .../test_bullet_lists.py | 116 +++-- .../test_recommonmark/test_enumerated_lists.py | 385 +++++++++++++++ .../test_recommonmark/test_html_blocks.py | 92 ++++ .../test_recommonmark/test_inline_markup.py | 539 +++++++++++++++++++++ .../test_line_length_limit.py} | 26 +- .../test_line_length_limit_default.py | 16 +- .../test_recommonmark/test_literal_blocks.py | 198 ++++++++ .../test_parsers/test_recommonmark/test_misc.py | 100 ++++ .../test_recommonmark/test_paragraphs.py | 81 ++++ .../test_recommonmark/test_section_headers.py | 268 ++++++++++ .../test_parsers/test_recommonmark/test_targets.py | 147 ++++++ .../test_transitions.py | 133 ++--- .../test_parsers/test_rst/test_bullet_lists.py | 12 +- .../test_rst/test_line_length_limit.py | 17 +- .../test_rst/test_line_length_limit_default.py | 17 +- .../test/test_parsers/test_rst/test_transitions.py | 13 +- docutils/tools/md2html5.py | 39 ++ docutils/tools/md2pseudoxml.py | 33 ++ docutils/tools/md2xml.py | 34 ++ 24 files changed, 2405 insertions(+), 165 deletions(-) create mode 100644 docutils/docutils/parsers/recommonmark_wrapper.py create mode 100644 docutils/test/test_parsers/test_recommonmark/__init__.py create mode 100755 docutils/test/test_parsers/test_recommonmark/test_block_quotes.py copy docutils/test/test_parsers/{test_rst => test_recommonmark}/test_bullet_lists.py (54%) create mode 100755 docutils/test/test_parsers/test_recommonmark/test_enumerated_lists.py create mode 100644 docutils/test/test_parsers/test_recommonmark/test_html_blocks.py create mode 100755 docutils/test/test_parsers/test_recommonmark/test_inline_markup.py copy docutils/test/test_parsers/{test_rst/test_line_length_limit_default.py => test_recommonmark/test_line_length_limit.py} (54%) mode change 100644 => 100755 copy docutils/test/test_parsers/{test_rst => test_recommonmark}/test_line_length_limit_default.py (62%) mode change 100644 => 100755 create mode 100755 docutils/test/test_parsers/test_recommonmark/test_literal_blocks.py create mode 100644 docutils/test/test_parsers/test_recommonmark/test_misc.py create mode 100755 docutils/test/test_parsers/test_recommonmark/test_paragraphs.py create mode 100755 docutils/test/test_parsers/test_recommonmark/test_section_headers.py create mode 100755 docutils/test/test_parsers/test_recommonmark/test_targets.py copy docutils/test/test_parsers/{test_rst => test_recommonmark}/test_transitions.py (71%) mode change 100644 => 100755 docutils/test/test_parsers/test_rst/test_line_length_limit.py mode change 100644 => 100755 docutils/test/test_parsers/test_rst/test_line_length_limit_default.py create mode 100755 docutils/tools/md2html5.py create mode 100644 docutils/tools/md2pseudoxml.py create mode 100644 docutils/tools/md2xml.py diff --git a/docutils/docutils/parsers/__init__.py b/docutils/docutils/parsers/__init__.py index 722af1e7f..3cfa86891 100644 --- a/docutils/docutils/parsers/__init__.py +++ b/docutils/docutils/parsers/__init__.py @@ -11,11 +11,27 @@ __docformat__ = 'reStructuredText' import sys from importlib import import_module -from docutils import Component +from docutils import Component, frontend class Parser(Component): + settings_spec = ( + 'Generic Parser Options', + None, + (('Disable the "raw" directives; replaced with a "warning" ' + 'system message.', + ['--no-raw'], + {'action': 'store_false', 'default': 1, 'dest': 'raw_enabled', + 'validator': frontend.validate_boolean}), + ('Enable the "raw" directive. Enabled by default.', + ['--raw-enabled'], + {'action': 'store_true'}), + ('Maximal number of characters in an input line. Default 10 000.', + ['--line-length-limit'], + {'metavar': '', 'type': 'int', 'default': 10000, + 'validator': frontend.validate_nonnegative_int}), + )) component_type = 'parser' config_section = 'parsers' @@ -39,7 +55,10 @@ _parser_aliases = { 'restructuredtext': 'rst', 'rest': 'rst', 'restx': 'rst', - 'rtxt': 'rst',} + 'rtxt': 'rst', + 'recommonmark': 'recommonmark_wrapper', + 'commonmark': 'recommonmark_wrapper', + 'markdown': 'recommonmark_wrapper'} def get_parser_class(parser_name): """Return the Parser class from the `parser_name` module.""" diff --git a/docutils/docutils/parsers/recommonmark_wrapper.py b/docutils/docutils/parsers/recommonmark_wrapper.py new file mode 100644 index 000000000..0864385c7 --- /dev/null +++ b/docutils/docutils/parsers/recommonmark_wrapper.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# -*- coding: utf8 -*- +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause +# +# Revision: $Revision$ +# Date: $Date$ +""" +A parser for CommonMark MarkDown text using `recommonmark`__. + +__ https://pypi.org/project/recommonmark/ +""" + +import docutils.parsers +from docutils import nodes + +try: + from recommonmark.parser import CommonMarkParser as _recommonmarkParser + _recommonmarkParser.supported = ('recommonmark', 'commonmark', + 'markdown', 'md') + with_recommonmark = True +except ImportError as err: + with_recommonmark = False + class _recommonmarkParser(docutils.parsers.Parser): + def parse(self, inputstring, document): + error = document.reporter.warning( + 'Missing dependency: MarkDown input is processed by a 3rd ' + 'party parser but Python did not find the required module ' + '"recommonmark" (https://pypi.org/project/recommonmark/).') + document.append(error) + +class Parser(_recommonmarkParser): + config_section = 'recommonmark parser' + config_section_dependencies = ('parsers',) + + def parse(self, inputstring, document): + """Use the upstream parser and clean up afterwards. + """ + # check for exorbitantly long lines + for i, line in enumerate(inputstring.split('\n')): + if len(line) > document.settings.line_length_limit: + error = document.reporter.error( + 'Line %d exceeds the line-length-limit.'%(i+1)) + document.append(error) + return + + # pass to upstream parser + try: + _recommonmarkParser.parse(self, inputstring, document) + except Exception as err: + error = document.reporter.error('Parsing with "recommonmark" ' + 'returned the error:\n%s'%err) + document.append(error) + + # Post-Processing + # --------------- + + # remove spurious empty lines + for node in document.traverse(nodes.TextElement): + node.children = [child for child in node.children + if not (isinstance(child, nodes.Text) + and str(child) == '\n')] + + # add "code" class argument to inline literal (code spans) + for node in document.traverse(lambda n: isinstance(n, + (nodes.literal, nodes.literal_block))): + node['classes'].append('code') + # move "language" argument to classes + for node in document.traverse(nodes.literal_block): + if 'language' in node.attributes: + node['classes'].append(node['language']) + del node['language'] + + # remove empty target nodes + for node in document.traverse(nodes.target): + # remove empty name + node['names'] = [v for v in node['names'] if v] + if node.children or [v for v in node.attributes.values() if v]: + continue + node.parent.remove(node) + + # replace raw nodes if raw is not allowed + if not document.settings.raw_enabled: + for node in document.traverse(nodes.raw): + warning = document.reporter.warning('Raw content disabled.') + node.parent.replace(node, warning) + + # fix section nodes + for node in document.traverse(nodes.section): + # remove spurious IDs (first may be from duplicate name) + if len(node['ids']) > 1: + node['ids'].pop() + # fix section levels + section_level = self.get_section_level(node) + if node['level'] != section_level: + warning = document.reporter.warning( + 'Title level inconsistent. Changing from %d to %d.' + %(node['level'], section_level), + nodes.literal_block('', node[0].astext())) + node.insert(1, warning) + # remove non-standard attribute "level" + del node['level'] # TODO: store the original md level somewhere + + def get_section_level(self, node): + level = 1 + while True: + node = node.parent + if isinstance(node, nodes.document): + return level + if isinstance(node, nodes.section): + level += 1 diff --git a/docutils/test/DocutilsTestSupport.py b/docutils/test/DocutilsTestSupport.py index dd902bd37..3b00a7fd2 100644 --- a/docutils/test/DocutilsTestSupport.py +++ b/docutils/test/DocutilsTestSupport.py @@ -47,6 +47,7 @@ import unittest import re import inspect import traceback +import warnings from pprint import pformat testroot = os.path.abspath(os.path.dirname(__file__) or os.curdir) @@ -63,7 +64,7 @@ try: from docutils import frontend, nodes, statemachine, utils from docutils.utils import urischemes from docutils.transforms import universal - from docutils.parsers import rst + from docutils.parsers import rst, recommonmark_wrapper from docutils.parsers.rst import states, tableparser, roles, languages from docutils.readers import standalone, pep from docutils.statemachine import StringList, string2lines @@ -82,7 +83,6 @@ try: except: import pdb - if sys.version_info >= (3, 0): unicode = str # noqa @@ -518,6 +518,38 @@ class PEPParserTestSuite(ParserTestSuite): test_case_class = PEPParserTestCase +class RecommonmarkParserTestCase(ParserTestCase): + + """Recommonmark-specific parser test case.""" + + parser = recommonmark_wrapper.Parser() + """Parser shared by all RecommonmarkParserTestCases.""" + + option_parser = frontend.OptionParser( + components=(recommonmark_wrapper.Parser,)) + settings = option_parser.get_default_values() + settings.report_level = 5 + settings.halt_level = 5 + settings.debug = package_unittest.debug + +class RecommonmarkParserTestSuite(ParserTestSuite): + + """A collection of RecommonmarkParserTestCases.""" + + test_case_class = RecommonmarkParserTestCase + skip_message = 'skip "%s" (module `recommonmark` not found)' + + def generateTests(self, dict, dictname='totest'): + if 'recommonmark' not in recommonmark_wrapper.Parser.supported: + if RecommonmarkParserTestSuite.skip_message: # warn (only once) + print(self.skip_message%self.id) + RecommonmarkParserTestSuite.skip_message = '' + return + # suppress UserWarnings from recommonmark parser + warnings.filterwarnings('ignore', message='Unsupported.*type') + ParserTestSuite.generateTests(self, dict, dictname='totest') + + class GridTableParserTestCase(CustomTestCase): parser = tableparser.GridTableParser() @@ -756,7 +788,7 @@ class HtmlWriterPublishPartsTestCase(WriterPublishTestCase): class HtmlPublishPartsTestSuite(CustomTestSuite): testcase_class = HtmlWriterPublishPartsTestCase - + def generateTests(self, dict, dictname='totest'): for name, (settings_overrides, cases) in dict.items(): settings = self.suite_settings.copy() diff --git a/docutils/test/test_parsers/test_recommonmark/__init__.py b/docutils/test/test_parsers/test_recommonmark/__init__.py new file mode 100644 index 000000000..0338a1f88 --- /dev/null +++ b/docutils/test/test_parsers/test_recommonmark/__init__.py @@ -0,0 +1,15 @@ +import os +import os.path +import sys +import unittest + +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) +prev = '' +while sys.path[0] != prev: + try: + import DocutilsTestSupport + break + except ImportError: + prev = sys.path[0] + sys.path[0] = os.path.dirname(prev) +sys.path.pop(0) diff --git a/docutils/test/test_parsers/test_recommonmark/test_block_quotes.py b/docutils/test/test_parsers/test_recommonmark/test_block_quotes.py new file mode 100755 index 000000000..1da33a9c6 --- /dev/null +++ b/docutils/test/test_parsers/test_recommonmark/test_block_quotes.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# -*- coding: utf8 -*- +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause + +""" +Test for block quotes in CommonMark parsers +Cf. the `CommonMark Specification `__ +""" + +from __future__ import absolute_import + +if __name__ == '__main__': + import __init__ +from test_parsers import DocutilsTestSupport + +def suite(): + s = DocutilsTestSupport.RecommonmarkParserTestSuite() + s.generateTests(totest) + return s + +totest = {} + +totest['block_quotes'] = [ +["""\ +> block quote +> line 2 +""", +"""\ + + + + block quote + line 2 +"""], +["""\ +Line 1. + + > Indented block quote. +""", +"""\ + + + Line 1. + + + Indented block quote. +"""], +["""\ +Line 1. +Line 2. +> Block quote, without blank line before. +""", +"""\ + + + Line 1. + Line 2. + + + Block quote, without blank line before. +"""], +["""\ +Line 1. +Line 2. + +>Block quote, +continuation line +""", +"""\ + + + Line 1. + Line 2. + + + Block quote, + continuation line +"""], +["""\ +Here is a paragraph. + +> > Nested +> +> block quotes. +""", +"""\ + + + Here is a paragraph. + + + + Nested + + block quotes. +"""], +] + + +if __name__ == '__main__': + import unittest + unittest.main(defaultTest='suite') diff --git a/docutils/test/test_parsers/test_rst/test_bullet_lists.py b/docutils/test/test_parsers/test_recommonmark/test_bullet_lists.py similarity index 54% copy from docutils/test/test_parsers/test_rst/test_bullet_lists.py copy to docutils/test/test_parsers/test_recommonmark/test_bullet_lists.py index 8817bbac9..93324d5b2 100755 --- a/docutils/test/test_parsers/test_rst/test_bullet_lists.py +++ b/docutils/test/test_parsers/test_recommonmark/test_bullet_lists.py @@ -1,12 +1,20 @@ -#! /usr/bin/env python - -# $Id$ -# Author: David Goodger -# Copyright: This module has been placed in the public domain. +#!/usr/bin/env python3 +# -*- coding: utf8 -*- +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause """ -Tests for states.py. +Test for bullet lists in CommonMark parsers. +Cf. the `CommonMark Specification `__ """ + from __future__ import absolute_import if __name__ == '__main__': @@ -14,7 +22,7 @@ if __name__ == '__main__': from test_parsers import DocutilsTestSupport def suite(): - s = DocutilsTestSupport.ParserTestSuite() + s = DocutilsTestSupport.RecommonmarkParserTestSuite() s.generateTests(totest) return s @@ -26,7 +34,7 @@ totest['bullet_lists'] = [ """, """\ - + item @@ -38,7 +46,7 @@ totest['bullet_lists'] = [ """, """\ - + item 1 @@ -56,7 +64,7 @@ No blank line between: No blank line between: - + item 1 @@ -65,20 +73,20 @@ No blank line between: item 2 """], ["""\ -- item 1, para 1. +- item 1, paragraph 1. - item 1, para 2. + item 1, paragraph 2. - item 2 """, """\ - + - item 1, para 1. + item 1, paragraph 1. - item 1, para 2. + item 1, paragraph 2. item 2 @@ -90,7 +98,7 @@ No blank line between: """, """\ - + item 1, line 1 @@ -100,54 +108,47 @@ No blank line between: item 2 """], ["""\ -Different bullets: +Different bullets start different lists: - item 1 -+ item 2 ++ item 1 -* item 3 -- item 4 +* no blank line +- required between lists """, """\ - Different bullets: - + Different bullets start different lists: + item 1 - + - item 2 - + item 1 + - item 3 - - - Bullet list ends without a blank line; unexpected unindent. - + no blank line + - item 4 + required between lists """], ["""\ -- item -no blank line +- item 1 +continuation of item 1 """, """\ - + - item - - - Bullet list ends without a blank line; unexpected unindent. - - no blank line + item 1 + continuation of item 1 """], ["""\ - @@ -156,7 +157,7 @@ empty item above """, """\ - + empty item above @@ -167,39 +168,30 @@ empty item above, no blank line """, """\ - + - - - Bullet list ends without a blank line; unexpected unindent. empty item above, no blank line """], [u"""\ -Unicode bullets: +Unicode bullets are not supported by CommonMark. -\u2022 BULLET +• BULLET -\u2023 TRIANGULAR BULLET +‣ TRIANGULAR BULLET -\u2043 HYPHEN BULLET +⁃ HYPHEN BULLET """, u"""\ - Unicode bullets: - - - - BULLET - - - - TRIANGULAR BULLET - - - - HYPHEN BULLET + Unicode bullets are not supported by CommonMark. + + • BULLET + + ‣ TRIANGULAR BULLET + + ⁃ HYPHEN BULLET """], ] diff --git a/docutils/test/test_parsers/test_recommonmark/test_enumerated_lists.py b/docutils/test/test_parsers/test_recommonmark/test_enumerated_lists.py new file mode 100755 index 000000000..bbf01148b --- /dev/null +++ b/docutils/test/test_parsers/test_recommonmark/test_enumerated_lists.py @@ -0,0 +1,385 @@ +#!/usr/bin/env python3 +# -*- coding: utf8 -*- +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause +""" +Test for enumerated lists in CommonMark parsers +Cf. the `CommonMark Specification `__ +""" +from __future__ import absolute_import + +if __name__ == '__main__': + import __init__ +from test_parsers import DocutilsTestSupport + +def suite(): + s = DocutilsTestSupport.RecommonmarkParserTestSuite() + s.generateTests(totest) + return s + +totest = {} + +totest['enumerated_lists'] = [ +["""\ +1. Item one. + +2. Item two. + +3. Item three. +""", +"""\ + + + + + Item one. + + + Item two. + + + Item three. +"""], +["""\ +No blank lines betwen items: + +1. Item one. +2. Item two. +3. Item three. +""", +"""\ + + + No blank lines betwen items: + + + + Item one. + + + Item two. + + + Item three. +"""], +["""\ +1. + Content may start at the next line + if it's indented at least 3 spaces. +""", +"""\ + + + + + Content may start at the next line + if it's indented at least 3 spaces. +"""], +["""\ +1. +empty item above, no blank line, no indent +""", +"""\ + + + + + empty item above, no blank line, no indent +"""], +["""\ +Items are auto-numbered. +No check for consistency: Skipping item 3 + +1. Item 1. +2. Item 2. +4. Item 4. +""", +"""\ + + + Items are auto-numbered. + No check for consistency: Skipping item 3 + + + + Item 1. + + + Item 2. + + + Item 4. +"""], +["""\ +No warning when starting with non-ordinal-1: + +0. Item zero. +1. Item one. +2. Item two. +3. Item three. + +And again: + +2. Item two. +3. Item three. +""", +"""\ + + + No warning when starting with non-ordinal-1: + + + + Item zero. + + + Item one. + + + Item two. + + + Item three. + + And again: + + + + Item two. + + + Item three. +"""], +["""\ +1. Item one: line 1, + line 2. +2. Item two: line 1, +line 2. +3. Item three: paragraph 1, line 1, + line 2. + + Paragraph 2. +""", +"""\ + + + + + Item one: line 1, + line 2. + + + Item two: line 1, + line 2. + + + Item three: paragraph 1, line 1, + line 2. + + Paragraph 2. +"""], +["""\ +Supported enumeration sequences: + +1. Item 1. +2. Item 2. + +1) Item 1) +2) Item 2) + +""", +"""\ + + + Supported enumeration sequences: + + + + Item 1. + + + Item 2. + + + + Item 1) + + + Item 2) +"""], +["""\ +Nested enumerated lists: + +1. Item 1 + + 1) Item 1.1 + 2) Item 1.2 + 3) Item 1.3 + +2. Item 2 + + 1. Item 2.1 + + 1) Item 2.1.1 + 2) Item 2.1.2 + 3) Item 2.1.3 + + 2. Item 2.2 + + 3. Item 2.3 + +3. Item 3. +""", +"""\ + + + Nested enumerated lists: + + + + Item 1 + + + + Item 1.1 + + + Item 1.2 + + + Item 1.3 + + + Item 2 + + + + Item 2.1 + + + + Item 2.1.1 + + + Item 2.1.2 + + + Item 2.1.3 + + + Item 2.2 + + + Item 2.3 + + + Item 3. +"""], +["""\ +1. Item one: line 1, + line 2. +2. Item two: line 1, + line 2. +3. Item three: paragraph 1, line 1, +line 2. + + Item three: paragraph 2. +""", +"""\ + + + + + Item one: line 1, + line 2. + + + Item two: line 1, + line 2. + + + Item three: paragraph 1, line 1, + line 2. + + Item three: paragraph 2. +"""], +["""\ +3-space indent, with a trailing space: + +1. \n\ + list item 1 + +3-space indent, no trailing space: + +1. + list item 1 + +2-space indent, empty list item: + +1. + foo + +1-space indent, empty list item: + +1. + foo + +0-space indent, empty list item: + +1. +foo + +No item content: + +1. +""", +"""\ + + + 3-space indent, with a trailing space: + + + + list item 1 + + 3-space indent, no trailing space: + + + + list item 1 + + 2-space indent, empty list item: + + + + foo + + 1-space indent, empty list item: + + + + foo + + 0-space indent, empty list item: + + + + foo + + No item content: + + +"""], +] + + +if __name__ == '__main__': + import unittest + unittest.main(defaultTest='suite') diff --git a/docutils/test/test_parsers/test_recommonmark/test_html_blocks.py b/docutils/test/test_parsers/test_recommonmark/test_html_blocks.py new file mode 100644 index 000000000..90481d0e4 --- /dev/null +++ b/docutils/test/test_parsers/test_recommonmark/test_html_blocks.py @@ -0,0 +1,92 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# $Id$ +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause +""" +Tests for HTML blocks in CommonMark parsers +Cf. the `CommonMark Specification `__ +""" +from __future__ import absolute_import + +if __name__ == '__main__': + import __init__ +from test_parsers import DocutilsTestSupport + + +def suite(): + s = DocutilsTestSupport.RecommonmarkParserTestSuite() + s.generateTests(totest) + return s + +totest = {} + +totest['html_blocks'] = [ +["""\ +A paragraph: + +

A HTML block.

+""", +"""\ + + + A paragraph: + +

A HTML block.

+"""], +["""\ +
+ +*Markdown* + +
+""", +"""\ + + +
+ + + Markdown + +
+"""], +["""\ + +*bar* + +""", +"""\ + + + + + + bar + + +"""], +["""\ +*bar* +*baz* +""", +"""\ + + + *bar* + *baz* +"""], +] + + + +if __name__ == '__main__': + import unittest + unittest.main(defaultTest='suite') diff --git a/docutils/test/test_parsers/test_recommonmark/test_inline_markup.py b/docutils/test/test_parsers/test_recommonmark/test_inline_markup.py new file mode 100755 index 000000000..ac6b6de37 --- /dev/null +++ b/docutils/test/test_parsers/test_recommonmark/test_inline_markup.py @@ -0,0 +1,539 @@ +#!/usr/bin/env python3 +# -*- coding: utf8 -*- +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause +""" +Tests for inline markup in CommonMark parsers +Cf. the `CommonMark Specification `__ +""" + +from __future__ import absolute_import + +if __name__ == '__main__': + import __init__ +from test_parsers import DocutilsTestSupport + + +def suite(): + s = DocutilsTestSupport.RecommonmarkParserTestSuite() + s.generateTests(totest) + return s + +totest = {} + +totest['emphasis'] = [ +["""\ +*emphasis* +_also emphasis_ +""", +"""\ + + + + emphasis + + also emphasis +"""], +[u"""\ +Partially*emphasised*word. +""", +u"""\ + + + Partially + + emphasised + word. +"""], +["""\ +*emphasized sentence +across lines* +""", +"""\ + + + + emphasized sentence + across lines +"""], +["""\ +*no emphasis without closing asterisk +""", +"""\ + + + * + no emphasis without closing asterisk +"""], +[r""" +No markup when \*escaped or unbalanced *. + +What about *this**? +Unbalanced _markup__ is kept as-is without warning. +""", +"""\ + + + No markup when \n\ + * + escaped or unbalanced \n\ + * + . + + What about \n\ + + this + * + ? + Unbalanced \n\ + + markup + _ + is kept as-is without warning. +"""], +[r""" +Emphasized asterisk: *\** + +Emphasized double asterisk: *\*\** +""", +"""\ + + + Emphasized asterisk: \n\ + + * + + Emphasized double asterisk: \n\ + + * + * +"""], +] + +totest['strong'] = [ +["""\ +**strong** +__also strong__ +""", +"""\ + + + + strong + + also strong +"""], +["""\ +Strong asterisk must be escaped **\\*** + +Strong double asterisk: **\\*\\*** +""", +"""\ + + + Strong asterisk must be escaped \n\ + + * + + Strong double asterisk: \n\ + + * + * +"""], +["""\ +**not strong without closing asterisks +""", +"""\ + + + ** + not strong without closing asterisks +"""], +] + +totest['literal'] = [ +["""\ +Inline `literals` are called `code spans` in CommonMark. +""", +"""\ + + + Inline \n\ + + literals + are called \n\ + + code spans + in CommonMark. +"""], +[r""" +`\*literal` +""", +"""\ + + + + \\*literal +"""], +[r""" +``lite\ral`` +""", +"""\ + + + + lite\\ral +"""], +[r""" +``literal\`` +""", +"""\ + + + + literal\\ +"""], +[u"""\ +l'``literal`` and l\u2019``literal`` with apostrophe +""", +u"""\ + + + l' + + literal + and l\u2019 + + literal + with apostrophe +"""], +[u"""\ +quoted '``literal``', quoted "``literal``", +quoted \u2018``literal``\u2019, quoted \u201c``literal``\u201d, +quoted \xab``literal``\xbb +""", +u"""\ + + + quoted ' + + literal + ', quoted " + + literal + ", + quoted \u2018 + + literal + \u2019, quoted \u201c + + literal + \u201d, + quoted \xab + + literal + \xbb +"""], +[u"""\ +``'literal'`` with quotes, ``"literal"`` with quotes, +``\u2018literal\u2019`` with quotes, ``\u201cliteral\u201d`` with quotes, +``\xabliteral\xbb`` with quotes +""", +u"""\ + + + + 'literal' + with quotes, \n\ + + "literal" + with quotes, + + \u2018literal\u2019 + with quotes, \n\ + + \u201cliteral\u201d + with quotes, + + \xabliteral\xbb + with quotes +"""], +[r""" +``literal ``no literal + +No warning for `standalone TeX quotes' or other *unbalanced markup**. +""", +"""\ + + + + literal + no literal + + No warning for \n\ + ` + standalone TeX quotes\' or other \n\ + + unbalanced markup + * + . +"""], +["""\ +``not literal without closing backquotes +""", +"""\ + + + `` + not literal without closing backquotes +"""], +[r""" +Python ``list``s use square bracket syntax. +""", +"""\ + + + Python \n\ + + list + s use square bracket syntax. +"""], +[r""" +Blank after opening `` not allowed. +""", +"""\ + + + Blank after opening \n\ + `` + not allowed. +"""], +[r""" +no blank ``after closing``still ends a literal. +""", +"""\ + + + no blank \n\ + + after closing + still ends a literal. +"""], +] + +totest['references'] = [ +["""\ +[ref] + +[ref]: /uri +""", +"""\ + + + + ref +"""], +["""\ +Inline image ![foo *bar*][foobar] +in a paragraph. + +[FOOBAR]: train.jpg "train & tracks" +""", +"""\ + + + Inline image \n\ + foo + in a paragraph. +"""], +["""\ +[phrase reference] + +[phrase reference]: /uri +""", +"""\ + + + + phrase reference +"""], +[u"""\ +No whitespace required around a[phrase reference]. + +[phrase reference]: /uri +""", +u"""\ + + + No whitespace required around a + + phrase reference + . +"""], +["""\ +[phrase reference +across lines] + +[phrase reference across lines]: /uri +""", +"""\ + + + + phrase reference + across lines +"""], +] + +totest['appended_URIs'] = [ +["""\ +[anonymous reference](http://example.com) +""", +"""\ + + + + anonymous reference +"""], +["""\ +Inline image ![a train](train.jpg) more text. +""", +"""\ + + + Inline image \n\ + a train + more text. +"""], +["""\ +Inline image ![foo](/url "title") more text. +""", +"""\ + + + Inline image \n\ + foo + more text. +"""], +["""\ +[URI must follow immediately] +(http://example.com) +""", +"""\ + + + [ + URI must follow immediately + ] + (http://example.com) +"""], +["""\ +Relative URIs' reference text can't be omitted: + +[reference](reference) +""", +"""\ + + + Relative URIs' reference text can't be omitted: + + + reference +"""], +] + +totest['standalone hyperlink'] = [ +["""\ +CommonMark calls standalone hyperlinks +like "autolinks". +""", +"""\ + + + CommonMark calls standalone hyperlinks + like \n\ + + http://example.com + "autolinks". +"""], +] + +totest['raw HTML'] = [ +["""\ +foo bar +""", +"""\ + + + foo \n\ + + + bar +"""], +["""\ +foo
bar +and +""", +"""\ + + + foo \n\ + +
+ bar + and \n\ + + +"""], +["""\ +Hard line breaks are not supported by Docutils. +Not the soft line break preceded by two or more spaces, \n\ +nor the more visible alternative,\\ +a backslash before the line ending. +""", +"""\ + + + Hard line breaks are not supported by Docutils. + Not the soft line break preceded by two or more spaces, + nor the more visible alternative, + a backslash before the line ending. +"""], +] + +totest['markup recognition rules'] = [ +[r""" +Character-level m*a***r**`k`_u_p +works except for underline. +""", +"""\ + + + Character-level m + + a + + r + + k + _ + u + _ + p + works except for underline. +"""], +] + + +if __name__ == '__main__': + import unittest + unittest.main(defaultTest='suite') diff --git a/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py b/docutils/test/test_parsers/test_recommonmark/test_line_length_limit.py old mode 100644 new mode 100755 similarity index 54% copy from docutils/test/test_parsers/test_rst/test_line_length_limit_default.py copy to docutils/test/test_parsers/test_recommonmark/test_line_length_limit.py index bbd49416e..1f82f9aa0 --- a/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py +++ b/docutils/test/test_parsers/test_recommonmark/test_line_length_limit.py @@ -1,9 +1,14 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# $Id$ -# Author: David Goodger -# Copyright: This module has been placed in the public domain. +#!/usr/bin/env python3 +# -*- coding: utf8 -*- +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause """ Tests for inline markup in docutils/parsers/rst/states.py. @@ -17,7 +22,8 @@ from test_parsers import DocutilsTestSupport def suite(): - s = DocutilsTestSupport.ParserTestSuite() + s = DocutilsTestSupport.RecommonmarkParserTestSuite( + suite_settings={'line_length_limit': 80}) s.generateTests(totest) return s @@ -27,17 +33,17 @@ totest['default'] = [ ["""\ within the limit %s -""" % ("x"*10000), +""" % ("x"*80), """\ within the limit %s -""" % ("x"*10000)], +""" % ("x"*80)], ["""\ above the limit %s -""" % ("x"*10001), +""" % ("x"*81), """\ diff --git a/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py b/docutils/test/test_parsers/test_recommonmark/test_line_length_limit_default.py old mode 100644 new mode 100755 similarity index 62% copy from docutils/test/test_parsers/test_rst/test_line_length_limit_default.py copy to docutils/test/test_parsers/test_recommonmark/test_line_length_limit_default.py index bbd49416e..e8f0560a0 --- a/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py +++ b/docutils/test/test_parsers/test_recommonmark/test_line_length_limit_default.py @@ -1,9 +1,15 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # -*- coding: utf-8 -*- - # $Id$ -# Author: David Goodger -# Copyright: This module has been placed in the public domain. +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause """ Tests for inline markup in docutils/parsers/rst/states.py. @@ -17,7 +23,7 @@ from test_parsers import DocutilsTestSupport def suite(): - s = DocutilsTestSupport.ParserTestSuite() + s = DocutilsTestSupport.RecommonmarkParserTestSuite() s.generateTests(totest) return s diff --git a/docutils/test/test_parsers/test_recommonmark/test_literal_blocks.py b/docutils/test/test_parsers/test_recommonmark/test_literal_blocks.py new file mode 100755 index 000000000..436902a2a --- /dev/null +++ b/docutils/test/test_parsers/test_recommonmark/test_literal_blocks.py @@ -0,0 +1,198 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# $Id$ +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause +""" +Tests for literal blocks in CommonMark parsers +Cf. the `CommonMark Specification `__ +""" +from __future__ import absolute_import + +if __name__ == '__main__': + import __init__ +from test_parsers import DocutilsTestSupport + + +def suite(): + s = DocutilsTestSupport.RecommonmarkParserTestSuite() + s.generateTests(totest) + return s + +totest = {} + +totest['literal_blocks'] = [ +["""\ +A paragraph: + + A literal block (indented code block). +""", +"""\ + + + A paragraph: + + A literal block (indented code block). +"""], +["""\ +A paragraph: +~~~ +A literal block (fenced code block). +~~~ +Another paragraph: + + Another literal block. + With two blank lines following. + + +A final paragraph. +""", +"""\ + + + A paragraph: + + A literal block (fenced code block). + + Another paragraph: + + Another literal block. + With two blank lines following. + + A final paragraph. +"""], +["""\ +A paragraph +on more than +one line. + No literal block + but paragraph continuation lines. +""", +"""\ + + + A paragraph + on more than + one line. + No literal block + but paragraph continuation lines. +"""], +["""\ +A paragraph: + + A literal block. +no blank line +""", +"""\ + + + A paragraph: + + A literal block. + + no blank line +"""], +["""\ +A paragraph: +``` + A fenced code block. +``` +no blank lines. +""", +"""\ + + + A paragraph: + + A fenced code block. + + no blank lines. +"""], +[r""" +A paragraph: + + Not a literal block because only indented 3 spaces. +""", +r""" + + A paragraph: + + Not a literal block because only indented 3 spaces. +"""], +[r""" + A literal block. + + Not a literal block. +""", +r""" + + A literal block. + + Not a literal block. +"""], +["""\ +A paragraph: + + A wonky literal block. + Literal line 2. + + Literal line 3. +""", +"""\ + + + A paragraph: + + A wonky literal block. + Literal line 2. + \n\ + Literal line 3. +"""], +["""\ +A paragraph: +~~~ + A fenced literal block. +Literal line 2. + + Literal line 3. +~~~ +""", +"""\ + + + A paragraph: + + A fenced literal block. + Literal line 2. + \n\ + Literal line 3. +"""], +["""\ +A paragraph: +~~~ ruby +A literal block (fenced code block) +with *info string*. +~~~ +""", +"""\ + + + A paragraph: + + A literal block (fenced code block) + with *info string*. +"""], +] + + + +if __name__ == '__main__': + import unittest + unittest.main(defaultTest='suite') diff --git a/docutils/test/test_parsers/test_recommonmark/test_misc.py b/docutils/test/test_parsers/test_recommonmark/test_misc.py new file mode 100644 index 000000000..6672bc7ca --- /dev/null +++ b/docutils/test/test_parsers/test_recommonmark/test_misc.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# -*- coding: utf8 -*- +# :Copyright: © 2020 Günter Milde. +# :License: Released under the terms of the `2-Clause BSD license`_, in short: +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# This file is offered as-is, without any warranty. +# +# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause + +""" +Various tests for the recommonmark parser. +""" + +from __future__ import absolute_import +import sys +import unittest + +if __name__ == '__main__': + import __init__ +from test_parsers import DocutilsTestSupport # must be imported before docutils +from docutils import core, utils +from docutils.core import publish_string +from docutils.parsers import recommonmark_wrapper + +sample1 = """\ +Test unexpected section titles. + +* Title + ===== + Paragraph. +""" + +sample_with_html = """\ +A paragraph: + +

A HTML block.

+ +Next paragraph. + +