new `Code2Text` converter after an idea by Riccardo Murri
[pylit.git] / test / pylit_test.py
blob53e77b31661fae6860c56b0ff4df87838789f4bf
1 #!/usr/bin/env python
2 # -*- coding: iso-8859-1 -*-
4 ## Test the pylit.py literal python module
5 ## =======================================
6 ##
7 ## :Version: 0.2
8 ## :Date: 2005-09-02
9 ## :Copyright: 2006 Guenter Milde.
10 ## Released under the terms of the GNU General Public License
11 ## (v. 2 or later)
13 ## .. contents::
15 ## ::
17 """pylit_test.py: test the "literal python" module"""
19 from pprint import pprint
20 from pylit import *
22 ## Text <-> Code conversion
23 ## ========================
25 ## Test strings
26 ## ------------
28 ## Example of text, code and stripped code with typical features"::
30 text = """.. #!/usr/bin/env python
31 # -*- coding: iso-8859-1 -*-
33 Leading text
35 in several paragraphs followed by a literal block::
37 block1 = 'first block'
39 Some more text and the next block. ::
41 block2 = 'second block'
42 print block1, block2
44 Trailing text.
45 """
46 # print text
48 ## The converter expects the data in separate lines (iterator or list)
49 ## with trailing newlines. We use the `splitlines` string method with
50 ## `keepends=True`::
52 textdata = text.splitlines(True)
53 # print textdata
55 ## If a "code source" is converted with the `strip` option, only text blocks
56 ## are extracted, which leads to::
58 stripped_text = """Leading text
60 in several paragraphs followed by a literal block:
62 Some more text and the next block.
64 Trailing text.
65 """
67 ## The code corresponding to the text test string.
69 ## Using a triple-quoted string for the code (and stripped_code) can create
70 ## problems with the conversion of this test by pylit (as the text parts
71 ## would be converted to text). This is catered for by using a different
72 ## comment string for the text blocks in this file: convert to text with
73 ## ``pylit --comment-string='## ' pylit_test.py``::
75 code = """#!/usr/bin/env python
76 # -*- coding: iso-8859-1 -*-
78 # Leading text
80 # in several paragraphs followed by a literal block::
82 block1 = 'first block'
84 # Some more text and the next block. ::
86 block2 = 'second block'
87 print block1, block2
89 # Trailing text.
90 """
91 # print code
93 codedata = code.splitlines(True)
95 ## Converting the text teststring with the `strip` option leads to::
97 stripped_code = """#!/usr/bin/env python
98 # -*- coding: iso-8859-1 -*-
100 block1 = 'first block'
102 block2 = 'second block'
103 print block1, block2
107 ## pprint(textdata)
108 ## pprint(stripped_code.splitlines(True))
110 ## Containers for special case examples:
112 ## 1. Text2Code samples
113 ## ``textsamples["what"] = (<text data>, <output>, <output (with `strip`)``
114 ## ::
116 textsamples = {}
118 ## 2. Code2Text samples
119 ## ``codesamples["what"] = (<code data>, <output>, <output (with `strip`)``
120 ## ::
122 codesamples = {}
124 ## Auxiliary function to test the textsamples and codesamples::
126 def check_converter(key, converter, output):
127 print "E:", key
128 extract = converter()
129 pprint(extract)
130 outstr = "".join(["".join(block) for block in extract])
131 print "soll:", repr(output)
132 print "ist: ", repr(outstr)
133 assert output == outstr
135 ## Test generator for textsample tests::
137 def test_Text2Code_samples():
138 for key, sample in textsamples.iteritems():
139 yield (check_converter, key,
140 Text2Code(sample[0].splitlines(True)), sample[1])
141 if len(sample) == 3:
142 yield (check_converter, key,
143 Text2Code(sample[0].splitlines(True), strip=True),
144 sample[2])
146 ## Test generator for codesample tests::
148 def test_Code2Text_samples():
149 for key, sample in codesamples.iteritems():
150 yield (check_converter, key,
151 Code2Text(sample[0].splitlines(True)), sample[1])
152 if len(sample) == 3:
153 yield (check_converter, key,
154 Code2Text(sample[0].splitlines(True), strip=True),
155 sample[2])
157 ## Text2Code
158 ## ---------
160 ## base tests on the "long" test data ::
162 def test_Text2Code():
163 """Test the Text2Code class converting rst->code"""
164 outstr = str(Text2Code(textdata))
165 print code,
166 print outstr
167 assert code == outstr
169 def test_Text2Code_strip():
170 """strip=True should strip text parts"""
171 outstr = str(Text2Code(textdata, strip=True))
172 print "ist ", repr(outstr)
173 print "soll", repr(stripped_code)
174 # pprint(outstr)
175 assert stripped_code == outstr
177 def test_Text2Code_malindented_code_line():
178 """raise error if code line is less indented than code-indent"""
179 data1 = [".. #!/usr/bin/env python\n", # indent == 4 * " "
180 "\n",
181 " print 'hello world'"] # indent == 2 * " "
182 data2 = ["..\t#!/usr/bin/env python\n", # indent == 4 * " "
183 "\n",
184 " print 'hello world'"] # indent == 2 * " "
185 for data in (data1, data2):
186 try:
187 blocks = Text2Code(data)()
188 assert False, "wrong indent did not raise ValueError"
189 except ValueError:
190 pass
192 ## Special Cases
193 ## ~~~~~~~~~~~~~
195 ## Code follows text block without blank line
196 ## ''''''''''''''''''''''''''''''''''''''''''
198 ## End of text block detected ('::') but no paragraph separator (blank line)
199 ## follows
201 ## It is an reStructuredText syntax error, if a "literal block
202 ## marker" is not followed by a blank line.
204 ## Assuming that no double colon at end of line occures accidentially,
205 ## pylit will fix this and issue a warning::
207 textsamples["ensure blank line after text"] = (
208 """text followed by a literal block::
209 block1 = 'first block'
210 """,
211 """# text followed by a literal block::
213 block1 = 'first block'
214 """)
216 ## Text follows code block without blank line
217 ## ''''''''''''''''''''''''''''''''''''''''''
219 ## End of code block detected (a line not more indented than the preceding text
220 ## block)
222 ## reStructuredText syntax demands a paragraph separator (blank line) before
223 ## it.
225 ## Assuming that the unindent is not accidential, pylit fixes this and issues a
226 ## warning::
228 textsamples["ensure blank line after code"] = (
229 """::
231 block1 = 'first block'
232 more text
233 """,
234 """# ::
236 block1 = 'first block'
238 # more text
239 """)
241 ## A double colon on a line on its own
242 ## '''''''''''''''''''''''''''''''''''
244 ## As a double colon is added by the Code2Text conversion after a text block
245 ## (if not already present), it could be removed by the Text2Code conversion
246 ## to keep the source small and pretty.
248 ## However, this would put the text and code source line numbers out of sync,
249 ## which is bad for error reporting, failing doctests, and the `pylit_buffer()`
250 ## function in http://jedmodes.sf.net/mode/pylit.sl ::
252 ## textsamples["should remove single double colon"] = (
253 ## ["text followed by a literal block\n",
254 ## "\n",
255 ## "::\n",
256 ## "\n",
257 ## " foo = 'first'\n"]
258 ## ["", # empty header
259 ## "# text followed by a literal block\n\n",
260 ## "foo = 'first'\n"]
262 ## header samples
263 ## ''''''''''''''
265 ## Convert a leading reStructured text comment (variant: only if there is
266 ## content on the first line) to a leading code block. Return an empty list,
267 ## if there is no header. ::
269 textsamples["simple header"] = (".. print 'hello world'",
270 "print 'hello world'")
272 textsamples["no header (start with text)"] = (
273 """a classical example without header::
275 print 'hello world'
276 """,
277 """# a classical example without header::
279 print 'hello world'
280 """)
282 textsamples["standard header, followed by text"] = (
283 """.. #!/usr/bin/env python
284 # -*- coding: iso-8859-1 -*-
286 a classical example with header::
288 print 'hello world'
289 """,
290 """#!/usr/bin/env python
291 # -*- coding: iso-8859-1 -*-
293 # a classical example with header::
295 print 'hello world'
296 """)
298 textsamples["standard header, followed by code"] = (
299 """.. #!/usr/bin/env python
301 print 'hello world'
302 """,
303 """#!/usr/bin/env python
305 print 'hello world'
306 """)
308 ## Code2Text
309 ## ---------
311 class test_Code2Text(object):
313 def setUp(self):
314 self.converter = Code2Text(codedata)
316 ## Code2Text.strip_literal_marker
318 ## * strip `::`-line as well as preceding blank line if on a line on its own
319 ## * strip `::` if it is preceded by whitespace.
320 ## * convert `::` to a single colon if preceded by text
322 def test_strip_literal_marker(self):
323 samples = (("text\n\n::\n\n", "text\n\n"),
324 ("text\n::\n\n", "text\n\n"),
325 ("text ::\n\n", "text\n\n"),
326 ("text::\n\n", "text:\n\n"),
327 ("text:\n\n", "text:\n\n"),
328 ("text\n\n", "text\n\n"),
329 ("text\n", "text\n")
331 for (ist, soll) in samples:
332 ist = ist.splitlines(True)
333 soll = soll.splitlines(True)
334 print "before", ist
335 self.converter.strip_literal_marker(ist)
336 print "soll:", repr(soll)
337 print "ist: ", repr(ist)
338 assert ist == soll
340 ## Code2Text.normalize_line
342 # Missing whitespace in the `comment_string` is not significant for otherwise
343 # blank lines. Add it::
345 def test_block_is_text(self):
346 samples = ((["code\n"], False),
347 (["#code\n"], False),
348 (["## code\n"], False),
349 (["# text\n"], True),
350 (["# text\n"], True),
351 (["# \n"], True),
352 (["#\n"], True),
353 (["\n"], True))
354 for (line, soll) in samples:
355 result = self.converter.block_is_text(line)
356 print repr(line), "soll", soll, "result", result
357 assert result == soll
359 ## base tests on the "long" test strings ::
361 def test_str(self):
362 """Test Code2Text class converting code->text"""
363 outstr = str(Code2Text(codedata))
364 # print text
365 print "soll:", repr(text)
366 print "ist: ", repr(outstr)
367 assert text == outstr
369 def test_str_strip(self):
370 """Test Code2Text class converting code->rst with strip=True
372 Should strip code blocks
374 pprint(Code2Text(codedata, strip=True)())
375 outstr = str(Code2Text(codedata, strip=True))
376 print repr(stripped_text)
377 print repr(outstr)
378 assert stripped_text == outstr
380 def test_str_different_comment_string(self):
381 """Convert only comments with the specified comment string to text
383 outstr = str(Code2Text(codedata, comment_string="##", strip=True))
384 print outstr
385 assert outstr == ""
386 data = ["# ::\n",
387 "\n",
388 "block1 = 'first block'\n",
389 "\n",
390 "## more text"]
391 soll = "\n".join(['.. # ::', # leading code block as header
392 ' ',
393 " block1 = 'first block'",
394 ' ',
395 ' more text'] # keep space (not part of comment string)
397 outstr = str(Code2Text(data, comment_string="##"))
398 print "soll:", repr(soll)
399 print "ist: ", repr(outstr)
400 assert outstr == soll
402 ## Special cases
403 ## ~~~~~~~~~~~~~
405 ## blank comment line
406 ## ''''''''''''''''''''
408 ## Normally, whitespace in the comment string is significant, i.e. with
409 ## `comment_string = "# "`, a line "#something\n" will count as code.
411 ## However, if a comment line is blank, trailing whitespace in the comment
412 ## string should be ignored, i.e. "#\n" is recognized as a blank text line::
414 codesamples["ignore trailing whitespace in comment string for blank line"] = (
415 """# ::
417 block1 = 'first block'
420 # more text
421 """,
422 """::
424 block1 = 'first block'
427 more text
428 """)
430 ## No blank line after text
431 ## ''''''''''''''''''''''''
433 ## If a matching comment precedes oder follows a code line (i.e. any line
434 ## without matching comment) without a blank line inbetween, it counts as code
435 ## line.
437 ## This will keep small inline comments close to the code they comment on. It
438 ## will also keep blocks together where one commented line doesnot match the
439 ## comment string (the whole block will be kept as commented code)
440 ## ::
442 codesamples["comment before code (without blank line)"] = (
443 """# this is text::
445 # this is a comment
446 foo = 'first'
447 """,
448 """this is text::
450 # this is a comment
451 foo = 'first'
452 """,
453 """this is text:
455 """)
457 codesamples["comment block before code (without blank line)"] = (
458 """# no text (watch the comment sign in the next line)::
460 # this is a comment
461 foo = 'first'
462 """,
463 """.. # no text (watch the comment sign in the next line)::
465 # this is a comment
466 foo = 'first'
467 """,
470 codesamples["comment after code (without blank line)"] = (
471 """# ::
473 block1 = 'first block'
474 # commented code
476 # text again
477 """,
478 """::
480 block1 = 'first block'
481 # commented code
483 text again
484 """,
486 text again
487 """)
489 codesamples["comment block after code (without blank line)"] = (
490 """# ::
492 block1 = 'first block'
493 # commented code
495 # still comment
496 """,
497 """::
499 block1 = 'first block'
500 # commented code
502 # still comment
503 """,
505 """)
507 ## missing literal block marker
508 ## ''''''''''''''''''''''''''''
510 ## If text (with matching comment string) is followed by code (line(s) without
511 ## matching comment string), but there is no double colon at the end, back
512 ## conversion would not recognize the end of text!
514 ## Therefore, pylit adds a paragraph containing only "::" -- the literal block
515 ## marker in expanded form. (While it would in many cases be nicer to add the
516 ## double colon to the last text line, this is not always valid rst syntax,
517 ## e.g. after a section header or a list. Therefore the automatic insertion
518 ## will use the save form, feel free to correct this by hand.)::
520 codesamples["insert missing double colon after text block"] = (
521 """# text followed by code without double colon
523 foo = 'first'
524 """,
525 """text followed by code without double colon
529 foo = 'first'
530 """,
531 """text followed by code without double colon
533 """)
535 ## header samples
536 ## ''''''''''''''
538 ## Convert a header (leading code block) to a reStructured text comment. ::
540 codesamples["no matching comment, just code"] = ("print 'hello world'",
541 ".. print 'hello world'")
543 codesamples["empty header (start with matching comment)"] = (
544 """# a classical example without header::
546 print 'hello world'
547 """,
548 """a classical example without header::
550 print 'hello world'
551 """,
552 """a classical example without header:
554 """)
556 codesamples["standard header, followed by text"] = (
557 """#!/usr/bin/env python
558 # -*- coding: iso-8859-1 -*-
560 # a classical example with header::
562 print 'hello world'
563 """,
564 """.. #!/usr/bin/env python
565 # -*- coding: iso-8859-1 -*-
567 a classical example with header::
569 print 'hello world'
570 """,
571 """a classical example with header:
573 """)
575 codesamples["standard header, followed by code"] = (
576 """#!/usr/bin/env python
578 print 'hello world'
579 """,
580 """.. #!/usr/bin/env python
582 print 'hello world'
583 """,
586 ## Command line use
587 ## ================
589 ## Test the option parsing::
591 def test_Values():
592 values = OptionValues()
593 print values
594 defaults = {"a1": 1, "a2": False}
595 values = OptionValues(defaults)
596 print values, values.as_dict()
597 assert values.a1 == 1
598 assert values.a2 == False
599 assert values.as_dict() == defaults
601 class test_PylitOptions:
602 """Test the PylitOption class"""
603 def setUp(self):
604 self.options = PylitOptions()
606 def test_languages_and_extensions(self):
607 """dictionary of programming languages and extensions"""
608 for ext in [".py", ".sl", ".c"]:
609 assert ext in self.options.code_extensions
610 assert self.options.code_languages[".py"] == "python"
611 assert self.options.code_languages[".sl"] == "slang"
612 assert self.options.code_languages[".c"] == "c++"
614 def test_parse_args(self):
615 """parse cmd line args"""
616 # default should appear in options
617 values = self.options.parse_args(txt2code=False)
618 print values, type(values), dir(values)
619 assert values.txt2code == False
620 # "cmd line arg should appear as option overwriting default"
621 values = self.options.parse_args(["--txt2code"], txt2code=False)
622 assert values.txt2code == True
623 # "1st non option arg is infile, 2nd is outfile"
624 values = self.options.parse_args(["--txt2code", "text.txt", "code.py"])
625 print values.infile
626 assert values.infile == "text.txt"
627 assert values.outfile == "code.py"
628 # set the output (option with argument)
629 values = self.options.parse_args(["--outfile", "code.py"])
630 assert values.outfile == "code.py"
632 def test_parse_args_comment_string(self):
633 # default should appear in options
634 values = self.options.parse_args(["--comment-string=% "])
635 pprint(values.as_dict())
636 assert values.comment_string == "% "
637 # "cmd line arg should appear as option overwriting default"
638 values = self.options.parse_args(["--comment-string=% "],
639 comment_string="##")
640 assert values.comment_string == '% '
642 def test_get_outfile_name(self):
643 """should return a sensible outfile name given an infile name"""
644 # return stdout for stdin
645 assert "-" == self.options.get_outfile_name("-")
646 # return with ".txt" stripped
647 assert "foo.py" == self.options.get_outfile_name("foo.py.txt")
648 # return with ".txt" added if extension marks code file
649 assert "foo.py.txt" == self.options.get_outfile_name("foo.py")
650 assert "foo.sl.txt" == self.options.get_outfile_name("foo.sl")
651 assert "foo.c.txt" == self.options.get_outfile_name("foo.c")
652 # return with ".txt" added if txt2code == False (not None!)
653 assert "foo.py.txt" == self.options.get_outfile_name("foo.py", txt2code=False)
654 # catchall: add ".out" if no other guess possible
655 assert "foo.out" == self.options.get_outfile_name("foo", txt2code=None)
657 def test_complete_values(self):
658 """Basic test of the option completion"""
659 values = optparse.Values()
660 values.infile = "foo"
661 values = self.options.complete_values(values)
662 # the following options should be set:
663 print values.infile # logo, as we give it...
664 print values.outfile
665 assert values.outfile == "foo.out" # fallback extension .out added
666 print values.txt2code
667 assert values.txt2code == True # the default
668 print values.language
669 assert values.language == "python" # the default
671 def test_complete_values_txt(self):
672 """Test the option completion with a text input file"""
673 values = optparse.Values()
674 values.infile = "foo.txt"
675 values = self.options.complete_values(values)
676 # should set outfile (see also `test_get_outfile_name`)
677 assert values.outfile == "foo"
678 # should set conversion direction according to extension
679 assert values.txt2code == True
681 def test_complete_values_code(self):
682 """Test the option completion with a code input file"""
683 values = optparse.Values()
684 values.infile = "foo.py"
685 values = self.options.complete_values(values)
686 # should set outfile name
687 assert values.outfile == "foo.py.txt"
688 # should set conversion directions according to extension
689 print values.txt2code
690 assert values.txt2code == False
692 def test_complete_values_dont_overwrite(self):
693 """The option completion must not overwrite existing option values"""
694 values = optparse.Values()
695 values.infile = "foo.py"
696 values.outfile = "bar.txt"
697 values.txt2code = True
698 values = self.options.complete_values(values)
699 assert values.outfile == "bar.txt"
700 assert values.txt2code == True
702 def test_init(self):
703 options = PylitOptions(["--txt2code", "foo"], txt2code=False)
704 pprint(options)
705 assert options.values.txt2code == True
706 assert options.values.infile == "foo"
708 ## Input and Output streams
709 ## ------------------------
711 ## ::
713 class IOTests:
714 """base class for IO tests, sets up and tears down example files in /tmp
716 txtpath = "/tmp/pylit_test.py.txt"
717 codepath = "/tmp/pylit_test.py"
718 outpath = "/tmp/pylit_test.out"
720 def setUp(self):
721 """Set up the test files"""
722 txtfile = file(self.txtpath, 'w')
723 txtfile.write(text)
724 # txtfile.flush() # is this needed if we close?
725 txtfile.close()
727 codefile = file(self.codepath, 'w')
728 codefile.write(code)
729 # codefile.flush() # is this needed if we close?
730 codefile.close()
732 def tearDown(self):
733 """clean up after all member tests are done"""
734 try:
735 os.unlink(self.txtpath)
736 os.unlink(self.codepath)
737 os.unlink(self.outpath)
738 except OSError:
739 pass
741 class test_Streams(IOTests):
742 def test_is_newer(self):
743 # this __file__ is older, than code file
744 print __file__, os.path.getmtime(__file__)
745 print self.codepath, os.path.getmtime(self.codepath)
747 assert is_newer(self.codepath, __file__) is True, "file1 is newer"
748 assert is_newer(__file__, self.codepath) is False, "file2 is newer"
749 assert is_newer(__file__, "fffo") is True, "file2 doesnot exist"
750 assert is_newer("fflo", __file__) is False, "file1 doesnot exist"
752 assert is_newer(__file__, __file__) is None, "equal is not newer"
753 assert is_newer("fflo", "fffo") is None, "no file exists -> equal"
755 def test_open_streams(self):
756 # default should return stdin and -out:
757 (instream, outstream) = open_streams()
758 assert instream is sys.stdin
759 assert outstream is sys.stdout
761 # open input and output file
762 (instream, outstream) = open_streams(self.txtpath, self.outpath)
763 assert type(instream) == file
764 assert type(outstream) == file
765 # read something from the input
766 assert instream.read() == text
767 # write something to the output
768 outstream.write(text)
769 # check the output, we have to flush first
770 outstream.flush()
771 outfile = file(self.outpath, 'r')
772 assert outfile.read() == text
774 def test_open_streams_no_infile(self):
775 """should exit with usage info if no infile given"""
776 try:
777 (instream, outstream) = open_streams("")
778 assert False, "should rise SystemExit"
779 except IOError:
780 pass
782 ## Another convenience function that returns a converter instance::
784 def test_get_converter():
785 # with default or txt2code
786 converter = get_converter(textdata)
787 print converter.__class__
788 assert converter.__class__ == Text2Code
789 converter = get_converter(textdata, txt2code=False)
790 assert converter.__class__ == Code2Text
792 # the run_doctest runs a doctest on the text version (as doc-string)
793 class test_Run_Doctest(IOTests):
794 """Doctest should run on the text source"""
795 def test_doctest_txt2code(self):
796 (failures, tests) = run_doctest(self.txtpath, txt2code=True)
797 assert (failures, tests) == (0, 0)
798 def test_doctest_code2txt(self):
799 (failures, tests) = run_doctest(self.codepath, txt2code=False)
800 assert (failures, tests) == (0, 0)
802 ## The main() function is called if the script is run from the command line
804 ## ::
806 class test_Main(IOTests):
807 """test default operation from command line
809 def get_output(self):
810 """read and return the content of the output file"""
811 outstream = file(self.outpath, 'r')
812 return outstream.read()
814 def test_text_to_code(self):
815 """test conversion of text file to code file"""
816 main(infile=self.txtpath, outfile=self.outpath)
817 output = self.get_output()
818 print repr(output)
819 assert output == code
821 def test_text_to_code_strip(self):
822 """test conversion of text file to stripped code file"""
823 main(infile=self.txtpath, outfile=self.outpath, strip=True)
824 output = self.get_output()
825 print repr(output)
826 assert output == stripped_code
828 def test_main_code_to_text(self):
829 """test conversion of code file to text file"""
830 main(infile=self.codepath, outfile=self.outpath)
831 output = self.get_output()
832 assert output == text
834 def test_main_code_to_text_strip(self):
835 """test conversion of code file to stripped text file"""
836 main(infile=self.codepath, outfile=self.outpath, strip=True)
837 output = self.get_output()
838 assert output == stripped_text
840 def test_main_diff(self):
841 result = main(infile=self.codepath, diff=True)
842 print "diff return value", result
843 assert result is False # no differences found
845 def test_main_diff_with_differences(self):
846 """diffing a file to itself should fail, as the input is converted"""
847 result = main(infile=self.codepath, outfile=self.codepath, diff=True)
848 print "diff return value", result
849 assert result is True # differences found
851 def test_main_execute(self):
852 result = main(infile=self.txtpath, execute=True)
853 print result
855 def test_main_execute_code(self):
856 result = main(infile=self.codepath, execute=True)
858 import nose
859 nose.runmodule() # requires nose 0.9.1
860 sys.exit()