sandbox/paultremblay/python_interface/test.py

   1 import sys, os, glob, commands, shlex, subprocess, argparse
   2 import docutilsToFo.rst2xml_lib
   3 from lxml import etree
   4 import lxml
   5 from docutilsToFo.rst2xml_lib import transform_lxml, publish_xml_cmdline, validate_fo_xsl
   6
   7 # $Id$
   8 TEST = False
   9 STRICT = True
  10
  11 parser = argparse.ArgumentParser(description='Test all the files')
  12 parser.add_argument('--xslt', nargs=1, choices = ['xsltproc', 'lxml', 'saxon', 'xalan'], default=['lxml'],
  13         dest='xsl_transform', help = 'choose which processer to use when transforming' )
  14 parser.add_argument('--pdf', action="store_const", const=True, help = 'convert to PDF', dest='pdf')
  15 parser.add_argument('--no-rst', action="store_const", const=True, help = 'don\'t convert to RST', dest='no_rst')
  16 parser.add_argument('--no-fo', action="store_const", const=True, help = 'don\'t convert to FO', dest='no_fo')
  17 parser.add_argument('--strict', nargs=1, choices = ['True', 'False'],
  18         default = 'True', help = 'whether to quit on errors', dest='strict')
  19 parser.add_argument('--debug', action="store_const", const=True, help = 'do debug messaging', dest='debug')
  20 parser.add_argument('--verbose', action="store_const", const=True, help = 'be verose in messaging', dest='verbose')
  21 parser.add_argument('--clean', action="store_const", const=True, help = 'whether to remove files after test', dest='clean')
  22 parser.add_argument('--full', action="store_const", const=True, help = 'Do a full test (for developer only)', dest='full')
  23
  24 arg = parser.parse_args()
  25 PDF = arg.pdf
  26 if  arg.strict == 'True':
  27     STRICT = True
  28 else:
  29     STRICT = False
  30 debug = False
  31 debug = arg.debug
  32 verbose = arg.verbose
  33 if debug: verbose = True
  34 FULL = False
  35 if arg.full:
  36     FULL = True
  37     PDF = True
  38
  39 # =========================================================================================
  40 # stylesheet  test
  41 # =========================================================================================
  42 test_dict = {
  43         'long_plain.xml':[({'page-layout':'simple'}, 'simple_no_page_nos.fo'),
  44                 ({'page-layout': 'first'}, 'first_page_diff_no_page_nos.fo'),
  45                 ({'page-layout': 'odd-even'}, 'odd_even_no_page_nos.fo'),
  46                 ({'page-layout': 'first-odd-even'}, 'first_odd_even_no_page_nos.fo')
  47                 ],
  48         'header_footer.xml':[({'page-layout': 'simple'}, 'header_footer.fo'),
  49                 ({'page-layout': 'simple', 'suppress-first-page-header': 'True'}, 'header_footer2.fo'),
  50                 ({'page-layout': 'first' }, 'first_page_header_footer.fo'),
  51                 ({'page-layout': 'first', 'suppress-first-page-header': 'True'}, 'first_page_suppress_header.fo'),
  52                 ({'page-layout': 'first', 'suppress-first-page-header': 'True', 'suppress-first-page-header': 'True'},
  53                     'first_suppress_header_footer.fo'),
  54                 ({'page-layout': 'odd-even' }, 'odd_even_page_header_footer.fo'),
  55                 ({'page-layout': 'first-odd-even' }, 'first-odd_even_page_header_footer.fo'),
  56                 ({'page-layout': 'first-odd-even', 'suppress-first-page-header': 'True', 'suppress-first-page-header': 'True'},
  57                     'first_odd_even_page_header_footer_suppress_first.fo'),
  58                 ],
  59         'opt_list.xml':[({},'opt_list.fo'),
  60                 ({'option-list-format': 'definition'}, 'opt_list_as_def.fo')
  61                 ],
  62         'table_csv.xml':[({},'table_csv.fo'),
  63                 ({'table-title-placement':'top'},'table_caption_top_csv.fo')
  64                 ],
  65         'footnotes.xml':[({},'footnotes.fo'),
  66                 ({'footnote-style': 'traditional'},'footnotes_traditional.fo')
  67                 ],
  68         'endnotes.xml':[({'footnote-placement':'endnote'}, 'endnotes.fo')
  69                 ],
  70         'hyperlinks.xml':[({}, 'hyperlinks.fo'),
  71                 ({'internal-link-type':'page'}, 'hyperlinks_page.fo'),
  72                 ({'internal-link-type':'page-link'}, 'hyperlinks_link_page.fo')
  73                 ],
  74
  75         'title_subtitle.xml':[({}, 'title_subtitle.fo'),
  76                 ({'page-layout': 'first'}, 'title_subtitle_first.fo'),
  77                 ({'page-layout': 'odd-even'}, 'title_subtitle_odd_even.fo'),
  78                 ({'page-layout': 'first-odd-even'}, 'title_subtitle_first_odd_even.fo')
  79                 ],
  80
  81         'bibliographic_fields_toc.xml':[
  82                 ({},'bibliographic_fields_toc.fo'),
  83                 ({'page-layout':'first'},'bibliographic_fields_first_toc.fo'),
  84                 ({'page-layout':'odd-even'},'bibliographic_fields_odd_even_toc.fo'),
  85                 ({'page-layout':'first-odd-even'},'bibliographic_fields_first_odd_even_toc.fo'),
  86                 ],
  87         'front_body.xml':[({},'front_matter.fo'),
  88                     ({'title-pagination':'with-front',
  89                     'bibliographic-pagination':'with-front',
  90                     'dedication-pagination':'with-front',
  91                     'dedication-pagination':'with-front',
  92                     'abstract-pagination':'with-front',
  93                     'toc-pagination':'with-front'},
  94                     'front_matter2.fo'),
  95                     ({'title-pagination':'with-front',
  96                     'bibliographic-pagination':'with-front',
  97                     'dedication-pagination':'with-body',
  98                     'dedication-pagination':'with-front',
  99                     'abstract-pagination':'with-toc',
 100                     'toc-pagination':'with-front'},
 101                     'front_matter3.fo'),
 102                     ({'title-pagination':'with-body',
 103                     'bibliographic-pagination':'with-front',
 104                     'dedication-pagination':'with-toc',
 105                     'dedication-pagination':'with-front',
 106                     'abstract-pagination':'with-toc',
 107                     'toc-pagination':'with-body'},
 108                     'front_matter4.fo'),
 109                     ({'title-pagination':'with-body',
 110                     'bibliographic-pagination':'with-front',
 111                     'dedication-pagination':'with-toc',
 112                     'dedication-pagination':'with-front',
 113                     'abstract-pagination':'with-toc',
 114                     'toc-pagination':'with-body',
 115                     'front-order':'toc, abstract, dedication,,title, bibliographic'},
 116                     'front_matter5.fo'),
 117             ],
 118
 119         'table_borderless.xml':[({}, 'table_borderless.fo'),],
 120         'table_grid_class.xml':[({}, 'table_grid_class.fo'),],
 121         'table_long1.xml':[({}, 'table_long1.fo'),
 122                 ({'table-title-placement':'top'}, 'table_long1_top_title.fo'),
 123                 ({'table-title-placement':'top','long-rows-first-page':'8' }, 'table_long1_top_title_split.fo'),
 124
 125                 ],
 126         'table_long2.xml':[({}, 'table_long2.fo'),
 127                 ({'table-title-placement':'top'}, 'table_long2_top_title.fo'),
 128                 ({'table-title-placement':'top','long-rows-first-page':'8' }, 'table_long2_top_title_split.fo'),
 129                 ],
 130         }
 131 # =========================================================================================
 132 # Docutils_to_fo test
 133 # =========================================================================================
 134
 135 docfo_commands = [
 136         ('long_plain.xml', 'paper_size.conf'),
 137         ('long_plain.xml', 'margins_simple.conf'),
 138         ('long_plain.xml', 'margins_first_odd_even.conf'),
 139         ('long_plain.xml', 'header_footer2.conf'),
 140         ('long_plain.xml', 'header_footer3.conf'),
 141         ('simple.xml', 'paragraph1.conf'),
 142         ('simple.xml', 'paragraph2.conf'),
 143         ('simple.xml', 'paragraph3.conf'),
 144         ('simple2.xml', 'paragraph4.conf'), # long para should not break across page
 145         ('simple.xml', 'paragraph5.conf'), # first paragraph test
 146         ('front_body.xml', 'front1.conf'), # all front matter
 147         ('front_body.xml', 'front2.conf'), # change order of front matter
 148         ('title_subtitle.xml', 'title1.conf'), # title with front matter
 149         ('bibliographic_fields.xml', 'bibliographic_fields.conf'), # generic bibliographic fields
 150         ('bibliographic_fields.xml', 'bibliographic_fields2.conf'), # bibliographic fields as blocks
 151         ('bibliographic_fields.xml', 'dedication1.conf'), # generic dedication
 152         ('bibliographic_fields.xml', 'abstract1.conf'), # generic dedication
 153         ('toc.xml', 'toc1.conf'), # generic toc
 154         ('bibliographic_fields_toc.xml', 'page_number1.conf'), # page numbering
 155         ('section.xml', 'section1.conf'), # generic sections
 156         ('transition.xml', 'transition1.conf'), # transition
 157         ('bullet_list.xml', 'bullet_list1.conf'), # bullet list
 158         ('enumerated_list.xml', 'enumerated_list1.conf'), # enumerated list
 159         ('definition_list.xml', 'definition_list1.conf'), # definition list
 160         ('field_lists.xml', 'field_list1.conf'), # field list
 161         ('opt_list.xml', 'option_list1.conf'), # option list as list
 162         ('opt_list.xml', 'option_list2.conf'), # option list as definition list
 163         ('literal_block.xml', 'literal_block1.conf'), # literal block
 164         ('line_block.xml', 'line_block1.conf'), # line block
 165         ('header_footer2.xml', 'header_footer4.conf'), # first header and footer
 166         ('header_footer5.xml', 'header_footer5.conf'), # odd even
 167         ('header_footer5.xml', 'header_footer6.conf'), # odd even, suppress first
 168         ('header_footer_toc1.xml', 'header_footer_toc1.conf'), # first and other
 169         ('header_footer_toc4.xml', 'header_footer_toc2.conf'), # odd even,
 170         ('header_footer_toc3.xml', 'header_footer_toc3.conf'), # first odd even,
 171         ('header_footer_toc2.xml', 'header_footer_toc1.conf'), #  suppress first
 172         ('block.xml', 'block_quote1.conf'), #  block quote
 173         ('table_simple.xml', 'table1.conf'), #  default table
 174     ]
 175
 176 def error_func(msg, the_path = None):
 177     if the_path:
 178         sys.stderr.write('Problems with "%s"\n' % the_path)
 179     if type(msg) == type(''):
 180         sys.stderr.write(msg)
 181     if STRICT:
 182         sys.exit(1)
 183         pass
 184
 185 def transform_xsl(xsl_file, xml_file, param_dict = {}, out_file = None, xsl_transform = 'lxml'):
 186     if not out_file:
 187         base, ext = os.path.splitext(xml_file)
 188         out_file = '%s.fo' % (base)
 189     if xsl_transform == 'xsltproc':
 190         command = 'xsltproc -o %s ' % (out_file)
 191         params = param_dict.keys()
 192         for param in params:
 193             command += ' --stringparam %s "%s" ' % (param, param_dict[param])
 194         command += ' %s %s ' % (xsl_file, xml_file)
 195         if TEST:
 196             print command
 197         status, output = commands.getstatusoutput(command)
 198         if status:
 199             msg = 'error converting %s to FO\n' % (xml_file)
 200             msg += 'command = "%s" \n' % (command)
 201             msg += output
 202             msg += '\n'
 203             error_func(msg)
 204     elif xsl_transform == 'saxon':
 205         command = 'saxon.sh '
 206         command += ' -o %s' % (out_file)
 207         if STRICT:
 208             command += ' -warnings:fatal '
 209         command  += ' %s %s' % (xml_file, xsl_file)
 210         params = param_dict.keys()
 211         for param in params:
 212             command += ' %s="%s" ' % (param, param_dict[param])
 213         status, output = commands.getstatusoutput(command)
 214         if status:
 215             msg = 'error converting %s to Fo\n' % (xml_file)
 216             msg += 'command = "%s" \n' % (command)
 217             msg += output
 218             msg += '\n'
 219             error_func(msg)
 220     elif xsl_transform == 'xalan':
 221         command = 'xalan.sh '
 222         command += ' -OUT %s' % (out_file)
 223         command  += ' -IN %s -XSL %s' % (xml_file, xsl_file)
 224         params = param_dict.keys()
 225         for param in params:
 226             command += ' -PARAM %s "%s" ' % (param, param_dict[param])
 227         status, output = commands.getstatusoutput(command)
 228         if status:
 229             msg = 'error converting %s to Fo\n' % (xml_file)
 230             msg += 'command = "%s" \n' % (command)
 231             msg += output
 232             msg += '\n'
 233             error_func(msg)
 234     elif xsl_transform == 'lxml':
 235         error = transform_lxml(xslt_file = xsl_file, xml_file= xml_file, param_dict=param_dict, out_file= out_file)
 236         if error:
 237             error_func(error, xml_file)
 238     else:
 239         sys.stderr.write('"%s" not a valid xsl_transform. Fix. Script now quitting\n' % (xsl_transform))
 240         sys.exit(1)
 241
 242
 243 def convert_to_xml(path_list):
 244     if verbose:
 245         sys.stderr.write('converting to xml...\n')
 246     num_files = len(path_list)
 247     counter = 0
 248     for the_path in path_list:
 249         counter += 1
 250         base, ext = os.path.splitext(the_path)
 251         out_file = '%s.xml' % (base)
 252         if verbose:
 253             sys.stderr.write('converting "%s" to "%s"\n' % (the_path, out_file))
 254         publish_xml_cmdline (in_path = the_path, out_path = out_file)
 255
 256 # simple, fail-proof method
 257 def convert_to_xml_command(path_list):
 258     # print 'converting to xml...'
 259     num_files = len(path_list)
 260     counter = 0
 261     for the_path in path_list:
 262         counter += 1
 263         # print 'converting %s of %s files' % (counter, num_files)
 264         base, ext = os.path.splitext(the_path)
 265         out_file = '%s.xml' % (base)
 266         command = 'rst2xml.py --trim-footnote-reference-space %s %s' % (the_path, out_file)
 267         status, output = commands.getstatusoutput(command)
 268
 269 def convert_to_fo(xsl_transform):
 270     if verbose:
 271         sys.stderr.write('converting to fo...\n')
 272     xml_files =  glob.glob('*.xml')
 273     len_simple = len(xml_files)
 274     the_keys = test_dict.keys()
 275     for the_key in the_keys:
 276         if the_key not in xml_files:
 277             sys.stderr.write('%s  not found in test_files; fix. Script now quitting' % (the_key))
 278             sys.exit(1)
 279     len_complex = len(the_keys)
 280     len_inside = 0
 281     for the_key in the_keys:
 282         this_inside = len(test_dict[the_key])
 283         len_inside += this_inside
 284     num_files = len_simple - len_complex + len_inside
 285     counter = 0
 286     for xml_file in xml_files:
 287         params = {'strict':'True'}
 288         transform_info = test_dict.get(xml_file)
 289         if transform_info:
 290             for info in transform_info:
 291                 counter += 1
 292                 if verbose:
 293                     sys.stderr.write('converting "%s" to %s to FO\n' % (xml_file, out_file))
 294                 # print 'converting %s of %s files' % (counter, num_files)
 295                 added_params = info[0]
 296                 out_file = info[1]
 297                 params.update(added_params)
 298                 transform_xsl(xsl_file = '../docutilsToFo/xsl_fo/docutils_to_fo.xsl',
 299                         xml_file = xml_file, param_dict= params, out_file = out_file,
 300                         xsl_transform = xsl_transform)
 301
 302                 validate_the_fo(out_file)
 303
 304         else:
 305             counter += 1
 306             # print 'converting %s of %s files' % (counter, num_files)
 307             base, ext = os.path.splitext(xml_file)
 308             out_file = '%s.fo' % (base)
 309             if verbose:
 310                 sys.stderr.write('converting "%s" to %s FO\n' % (xml_file, out_file))
 311             transform_xsl(xsl_file = '../docutilsToFo/xsl_fo/docutils_to_fo.xsl', xml_file = xml_file, param_dict=params, xsl_transform = xsl_transform)
 312             validate_the_fo(out_file)
 313
 314 def convert_to_pdf():
 315     # print 'converting to pdf...'
 316     fo_files =  glob.glob('*.fo')
 317     num_files = len(fo_files)
 318     counter = 0
 319     for fo_file in fo_files:
 320         counter += 1
 321         # print 'converting %s of %s files' % (counter, num_files)
 322         base, ext = os.path.splitext(fo_file)
 323         out_file = '%s.pdf' % (base)
 324         command = 'fop  %s %s' % (fo_file, out_file)
 325         status, output = commands.getstatusoutput(command)
 326         if status:
 327             msg = 'error converting %s to PDF\n' % (fo_file)
 328             msg += 'command = "%s" \n' % (command)
 329             msg += output
 330             msg += '\n'
 331             error(msg)
 332
 333 def convert_fo_to_pdf(fo_file, out_file):
 334     command = 'fop  %s %s' % (fo_file, out_file)
 335     if verbose:
 336         sys.stderr.write('\n doing command "%s"\n' % (command))
 337     status, output = commands.getstatusoutput(command)
 338     if status:
 339         msg = 'error converting %s to PDF\n' % (fo_file)
 340         msg += 'command = "%s" \n' % (command)
 341         msg += output
 342         msg += '\n'
 343         error(msg)
 344
 345
 346 # not used
 347 def make_conf_file(the_dict, section = 'FO', conf_path = 'docutils.conf'):
 348     rm_path(conf_path)
 349     write_obj = file(conf_path, 'w')
 350     the_keys = the_dict.keys()
 351     write_obj.write('[%s]\n' % section)
 352     for the_key in the_keys:
 353         write_obj.write('%s = %s\n' % (the_key, the_dict[the_key]))
 354     write_obj.close()
 355     return conf_path
 356
 357 def run_docutils_command(command):
 358     status, output = commands.getstatusoutput(command)
 359     if status:
 360         msg = 'error converting  to FO with docutilst_to_fo.py:\n'
 361         msg += 'command = "%s" \n' % (command)
 362         msg += output
 363         msg += '\n'
 364         error_func(msg)
 365
 366 def validate_the_fo(xml_file):
 367     error = validate_fo_xsl(xml_file)
 368     if error:
 369         sys.stderr.write('Problems converting "%s"\n' % (xml_file))
 370         error_func('')
 371
 372 def test_docutils_to_fo_script(script_command):
 373     if verbose:
 374         sys.stderr.write('\n\n==========================================\n')
 375         sys.stderr.write('TESTING DOCUTILS SCRIPT\n')
 376         sys.stderr.write('\n==========================================\n')
 377     command = '%s -h' % script_command
 378     status, output = commands.getstatusoutput(command)
 379     default__commands = ' --config docutils.conf '
 380     for info in docfo_commands:
 381         command = script_command
 382         in_file = info[0]
 383         filename,ext = os.path.splitext(in_file)
 384         out_file = '%s.fo' % filename
 385         out_opt = ' -o %s ' % out_file
 386         config_file = os.path.join('conf_files', info[1])
 387         if not os.path.isfile(config_file):
 388             sys.stderr.write('Can\'t find "%s" conf file\n' % config_file)
 389             sys.exit(1)
 390         config_opt = ' --config %s ' % config_file
 391         command += ' %s %s %s' % (out_opt, config_opt, in_file)
 392         if verbose:
 393             sys.stderr.write('Converting "%s" to "%s" with "%s" \n' % (in_file, out_file, config_file))
 394         run_docutils_command(command)
 395         validate_the_fo(out_file)
 396         if PDF:
 397             out_pdf = '%s.pdf' % filename
 398             convert_fo_to_pdf(fo_file = out_file, out_file = out_pdf)
 399
 400
 401 def clean():
 402     remove_paths(glob.glob('*.xml'))
 403     remove_paths(glob.glob('*.xsl'))
 404     remove_paths(glob.glob('*.fo'))
 405     remove_paths(glob.glob('*.pdf'))
 406
 407 def rm_path(the_path):
 408     try:
 409         os.remove(the_path)
 410     except OSError:
 411         pass
 412
 413 def remove_paths(list_of_paths):
 414     for the_path in list_of_paths:
 415         rm_path(the_path)
 416
 417 def full():
 418     # main()
 419     if verbose:
 420         sys.stderr.write('In FULL mode\n')
 421     clean()
 422     script_path  = os.path.abspath(os.path.join('scripts', 'docutils_to_fo.py'))
 423     script_command = 'python %s ' % script_path
 424     current_dir = os.getcwd()
 425     os.chdir('test_files')
 426     rst_files = glob.glob('*.rst')
 427     convert_to_xml(rst_files)
 428     xsl_trans = ['xsltproc', 'saxon',  'xalan', 'lxml',]
 429     for xsl_tran in xsl_trans:
 430         if verbose:
 431             sys.stderr.write('Transforming to FO with %s\n' % (xsl_tran))
 432         convert_to_fo(xsl_transform = xsl_tran)
 433         if xsl_tran != 'lxml':
 434             remove_paths(glob.glob('*.fo'))
 435     convert_to_pdf()
 436     test_docutils_to_fo_script(script_command)
 437     clean()
 438     os.chdir(current_dir)
 439     if verbose:
 440         sys.stderr.write('\n\nSUCCESS!\n')
 441
 442
 443 def main():
 444     script_path  = os.path.abspath(os.path.join('scripts', 'docutils_to_fo.py'))
 445     script_command = 'python %s ' % script_path
 446     current_dir = os.getcwd()
 447     os.chdir('test_files')
 448     if not arg.no_rst:
 449         rst_files = glob.glob('*.rst')
 450         convert_to_xml(rst_files)
 451     if not arg.no_fo:
 452         convert_to_fo(xsl_transform = arg.xsl_transform[0])
 453     if PDF:
 454         convert_to_pdf()
 455     for fo_file in glob.glob('*.fo'):
 456         os.remove(fo_file)
 457     test_docutils_to_fo_script(script_command)
 458     #if arg.clean:
 459         # clean()
 460     # clean()
 461     os.chdir(current_dir)
 462
 463 try:
 464     import locale
 465     locale.setlocale(locale.LC_ALL, '')
 466 except:
 467     pass
 468
 469 if FULL:
 470     full()
 471 else:
 472     main()
 473