docutils/test/test_io.py

   1 #! /usr/bin/env python
   2
   3 # $Id$
   4 # Author: Lea Wiemann <LeWiemann@gmail.com>
   5 # Copyright: This module has been placed in the public domain.
   6
   7 """
   8 Test module for io.py.
   9 """
  10
  11 import unittest, sys
  12 import DocutilsTestSupport              # must be imported before docutils
  13 from docutils import io
  14 from docutils._compat import b, bytes
  15 from docutils.error_reporting import locale_encoding
  16
  17 class InputTests(unittest.TestCase):
  18
  19     # python 2.3
  20     if not hasattr(unittest.TestCase, "assertTrue"):
  21         assertTrue = unittest.TestCase.failUnless
  22
  23     def test_bom(self):
  24         input = io.StringInput(source=b('\xef\xbb\xbf foo \xef\xbb\xbf bar'),
  25                                encoding='utf8')
  26         # Assert BOMs are gone.
  27         self.assertEqual(input.read(), u' foo  bar')
  28         # With unicode input:
  29         input = io.StringInput(source=u'\ufeff foo \ufeff bar')
  30         # Assert BOMs are still there.
  31         self.assertEqual(input.read(), u'\ufeff foo \ufeff bar')
  32
  33     def test_coding_slug(self):
  34         input = io.StringInput(source=b("""\
  35 .. -*- coding: ascii -*-
  36 data
  37 blah
  38 """))
  39         data = input.read()
  40         self.assertEqual(input.successful_encoding, 'ascii')
  41         input = io.StringInput(source=b("""\
  42 #! python
  43 # -*- coding: ascii -*-
  44 print "hello world"
  45 """))
  46         data = input.read()
  47         self.assertEqual(input.successful_encoding, 'ascii')
  48         input = io.StringInput(source=b("""\
  49 #! python
  50 # extraneous comment; prevents coding slug from being read
  51 # -*- coding: ascii -*-
  52 print "hello world"
  53 """))
  54         data = input.read()
  55         self.assertNotEqual(input.successful_encoding, 'ascii')
  56
  57     def test_bom_detection(self):
  58         source = u'\ufeffdata\nblah\n'
  59         input = io.StringInput(source=source.encode('utf-16-be'))
  60         data = input.read()
  61         self.assertEqual(input.successful_encoding, 'utf-16-be')
  62         input = io.StringInput(source=source.encode('utf-16-le'))
  63         data = input.read()
  64         self.assertEqual(input.successful_encoding, 'utf-16-le')
  65         input = io.StringInput(source=source.encode('utf-8'))
  66         data = input.read()
  67         self.assertEqual(input.successful_encoding, 'utf-8')
  68
  69     def test_readlines(self):
  70         input = io.FileInput(source_path='data/include.txt')
  71         data = input.readlines()
  72         self.assertEqual(data, [u'Some include text.\n'])
  73
  74     def test_heuristics_utf8(self):
  75         # if no encoding is given, try decoding with utf8:
  76         input = io.FileInput(source_path='functional/input/cyrillic.txt')
  77         data = input.read()
  78         if sys.version_info < (3,0):
  79             # in Py3k, the locale encoding is used without --input-encoding
  80             # skipping the heuristic
  81             self.assertEqual(input.successful_encoding, 'utf-8')
  82
  83     def test_heuristics_no_utf8(self):
  84         # if no encoding is given and decoding with utf8 fails,
  85         # use either the locale encoding (if specified) or latin1:
  86         input = io.FileInput(source_path='data/latin1.txt')
  87         data = input.read()
  88         self.assertTrue(input.successful_encoding in (locale_encoding,
  89                                                       'latin-1'))
  90         if input.successful_encoding == 'latin-1':
  91             self.assertEqual(data, u'Gr\xfc\xdfe\n')
  92
  93
  94 if __name__ == '__main__':
  95     unittest.main()