1 import sys
, os
, glob
, commands
, shlex
, subprocess
, argparse
2 import docutilsToFo
.rst2xml_lib
5 from docutilsToFo
.rst2xml_lib
import transform_lxml
, publish_xml_cmdline
, validate_fo_xsl
11 parser
= argparse
.ArgumentParser(description
='Test all the files')
12 parser
.add_argument('--xslt', nargs
=1, choices
= ['xsltproc', 'lxml', 'saxon', 'xalan'], default
=['lxml'],
13 dest
='xsl_transform', help = 'choose which processer to use when transforming' )
14 parser
.add_argument('--pdf', action
="store_const", const
=True, help = 'convert to PDF', dest
='pdf')
15 parser
.add_argument('--no-rst', action
="store_const", const
=True, help = 'don\'t convert to RST', dest
='no_rst')
16 parser
.add_argument('--no-fo', action
="store_const", const
=True, help = 'don\'t convert to FO', dest
='no_fo')
17 parser
.add_argument('--strict', nargs
=1, choices
= ['True', 'False'],
18 default
= 'True', help = 'whether to quit on errors', dest
='strict')
19 parser
.add_argument('--debug', action
="store_const", const
=True, help = 'do debug messaging', dest
='debug')
20 parser
.add_argument('--verbose', action
="store_const", const
=True, help = 'be verose in messaging', dest
='verbose')
21 parser
.add_argument('--clean', action
="store_const", const
=True, help = 'whether to remove files after test', dest
='clean')
22 parser
.add_argument('--full', action
="store_const", const
=True, help = 'Do a full test (for developer only)', dest
='full')
24 arg
= parser
.parse_args()
26 if arg
.strict
== 'True':
33 if debug
: verbose
= True
39 # =========================================================================================
41 # =========================================================================================
43 'long_plain.xml':[({'page-layout':'simple'}, 'simple_no_page_nos.fo'),
44 ({'page-layout': 'first'}, 'first_page_diff_no_page_nos.fo'),
45 ({'page-layout': 'odd-even'}, 'odd_even_no_page_nos.fo'),
46 ({'page-layout': 'first-odd-even'}, 'first_odd_even_no_page_nos.fo')
48 'header_footer.xml':[({'page-layout': 'simple'}, 'header_footer.fo'),
49 ({'page-layout': 'simple', 'suppress-first-page-header': 'True'}, 'header_footer2.fo'),
50 ({'page-layout': 'first' }, 'first_page_header_footer.fo'),
51 ({'page-layout': 'first', 'suppress-first-page-header': 'True'}, 'first_page_suppress_header.fo'),
52 ({'page-layout': 'first', 'suppress-first-page-header': 'True', 'suppress-first-page-header': 'True'},
53 'first_suppress_header_footer.fo'),
54 ({'page-layout': 'odd-even' }, 'odd_even_page_header_footer.fo'),
55 ({'page-layout': 'first-odd-even' }, 'first-odd_even_page_header_footer.fo'),
56 ({'page-layout': 'first-odd-even', 'suppress-first-page-header': 'True', 'suppress-first-page-header': 'True'},
57 'first_odd_even_page_header_footer_suppress_first.fo'),
59 'opt_list.xml':[({},'opt_list.fo'),
60 ({'option-list-format': 'definition'}, 'opt_list_as_def.fo')
62 'table_csv.xml':[({},'table_csv.fo'),
63 ({'table-title-placement':'top'},'table_caption_top_csv.fo')
65 'footnotes.xml':[({},'footnotes.fo'),
66 ({'footnote-style': 'traditional'},'footnotes_traditional.fo')
68 'endnotes.xml':[({'footnote-placement':'endnote'}, 'endnotes.fo')
70 'hyperlinks.xml':[({}, 'hyperlinks.fo'),
71 ({'internal-link-type':'page'}, 'hyperlinks_page.fo'),
72 ({'internal-link-type':'page-link'}, 'hyperlinks_link_page.fo')
75 'title_subtitle.xml':[({}, 'title_subtitle.fo'),
76 ({'page-layout': 'first'}, 'title_subtitle_first.fo'),
77 ({'page-layout': 'odd-even'}, 'title_subtitle_odd_even.fo'),
78 ({'page-layout': 'first-odd-even'}, 'title_subtitle_first_odd_even.fo')
81 'bibliographic_fields_toc.xml':[
82 ({},'bibliographic_fields_toc.fo'),
83 ({'page-layout':'first'},'bibliographic_fields_first_toc.fo'),
84 ({'page-layout':'odd-even'},'bibliographic_fields_odd_even_toc.fo'),
85 ({'page-layout':'first-odd-even'},'bibliographic_fields_first_odd_even_toc.fo'),
87 'front_body.xml':[({},'front_matter.fo'),
88 ({'title-pagination':'with-front',
89 'bibliographic-pagination':'with-front',
90 'dedication-pagination':'with-front',
91 'dedication-pagination':'with-front',
92 'abstract-pagination':'with-front',
93 'toc-pagination':'with-front'},
95 ({'title-pagination':'with-front',
96 'bibliographic-pagination':'with-front',
97 'dedication-pagination':'with-body',
98 'dedication-pagination':'with-front',
99 'abstract-pagination':'with-toc',
100 'toc-pagination':'with-front'},
102 ({'title-pagination':'with-body',
103 'bibliographic-pagination':'with-front',
104 'dedication-pagination':'with-toc',
105 'dedication-pagination':'with-front',
106 'abstract-pagination':'with-toc',
107 'toc-pagination':'with-body'},
109 ({'title-pagination':'with-body',
110 'bibliographic-pagination':'with-front',
111 'dedication-pagination':'with-toc',
112 'dedication-pagination':'with-front',
113 'abstract-pagination':'with-toc',
114 'toc-pagination':'with-body',
115 'front-order':'toc, abstract, dedication,,title, bibliographic'},
119 'table_borderless.xml':[({}, 'table_borderless.fo'),],
120 'table_grid_class.xml':[({}, 'table_grid_class.fo'),],
121 'table_long1.xml':[({}, 'table_long1.fo'),
122 ({'table-title-placement':'top'}, 'table_long1_top_title.fo'),
123 ({'table-title-placement':'top','long-rows-first-page':'8' }, 'table_long1_top_title_split.fo'),
126 'table_long2.xml':[({}, 'table_long2.fo'),
127 ({'table-title-placement':'top'}, 'table_long2_top_title.fo'),
128 ({'table-title-placement':'top','long-rows-first-page':'8' }, 'table_long2_top_title_split.fo'),
131 # =========================================================================================
132 # Docutils_to_fo test
133 # =========================================================================================
136 ('long_plain.xml', 'paper_size.conf'),
137 ('long_plain.xml', 'margins_simple.conf'),
138 ('long_plain.xml', 'margins_first_odd_even.conf'),
139 ('long_plain.xml', 'header_footer2.conf'),
140 ('long_plain.xml', 'header_footer3.conf'),
141 ('simple.xml', 'paragraph1.conf'),
142 ('simple.xml', 'paragraph2.conf'),
143 ('simple.xml', 'paragraph3.conf'),
144 ('simple2.xml', 'paragraph4.conf'), # long para should not break across page
145 ('simple.xml', 'paragraph5.conf'), # first paragraph test
146 ('front_body.xml', 'front1.conf'), # all front matter
147 ('front_body.xml', 'front2.conf'), # change order of front matter
148 ('title_subtitle.xml', 'title1.conf'), # title with front matter
149 ('bibliographic_fields.xml', 'bibliographic_fields.conf'), # generic bibliographic fields
150 ('bibliographic_fields.xml', 'bibliographic_fields2.conf'), # bibliographic fields as blocks
151 ('bibliographic_fields.xml', 'dedication1.conf'), # generic dedication
152 ('bibliographic_fields.xml', 'abstract1.conf'), # generic dedication
153 ('toc.xml', 'toc1.conf'), # generic toc
154 ('bibliographic_fields_toc.xml', 'page_number1.conf'), # page numbering
155 ('section.xml', 'section1.conf'), # generic sections
156 ('transition.xml', 'transition1.conf'), # transition
157 ('bullet_list.xml', 'bullet_list1.conf'), # bullet list
158 ('enumerated_list.xml', 'enumerated_list1.conf'), # enumerated list
159 ('definition_list.xml', 'definition_list1.conf'), # definition list
160 ('field_lists.xml', 'field_list1.conf'), # field list
161 ('opt_list.xml', 'option_list1.conf'), # option list as list
162 ('opt_list.xml', 'option_list2.conf'), # option list as definition list
163 ('literal_block.xml', 'literal_block1.conf'), # literal block
164 ('line_block.xml', 'line_block1.conf'), # line block
165 ('header_footer2.xml', 'header_footer4.conf'), # first header and footer
166 ('header_footer5.xml', 'header_footer5.conf'), # odd even
167 ('header_footer5.xml', 'header_footer6.conf'), # odd even, suppress first
168 ('header_footer_toc1.xml', 'header_footer_toc1.conf'), # first and other
169 ('header_footer_toc4.xml', 'header_footer_toc2.conf'), # odd even,
170 ('header_footer_toc3.xml', 'header_footer_toc3.conf'), # first odd even,
171 ('header_footer_toc2.xml', 'header_footer_toc1.conf'), # suppress first
172 ('block.xml', 'block_quote1.conf'), # block quote
173 ('table_simple.xml', 'table1.conf'), # default table
176 def error_func(msg
, the_path
= None):
178 sys
.stderr
.write('Problems with "%s"\n' % the_path
)
179 if type(msg
) == type(''):
180 sys
.stderr
.write(msg
)
185 def transform_xsl(xsl_file
, xml_file
, param_dict
= {}, out_file
= None, xsl_transform
= 'lxml'):
187 base
, ext
= os
.path
.splitext(xml_file
)
188 out_file
= '%s.fo' % (base
)
189 if xsl_transform
== 'xsltproc':
190 command
= 'xsltproc -o %s ' % (out_file
)
191 params
= param_dict
.keys()
193 command
+= ' --stringparam %s "%s" ' % (param
, param_dict
[param
])
194 command
+= ' %s %s ' % (xsl_file
, xml_file
)
197 status
, output
= commands
.getstatusoutput(command
)
199 msg
= 'error converting %s to FO\n' % (xml_file
)
200 msg
+= 'command = "%s" \n' % (command
)
204 elif xsl_transform
== 'saxon':
205 command
= 'saxon.sh '
206 command
+= ' -o %s' % (out_file
)
208 command
+= ' -warnings:fatal '
209 command
+= ' %s %s' % (xml_file
, xsl_file
)
210 params
= param_dict
.keys()
212 command
+= ' %s="%s" ' % (param
, param_dict
[param
])
213 status
, output
= commands
.getstatusoutput(command
)
215 msg
= 'error converting %s to Fo\n' % (xml_file
)
216 msg
+= 'command = "%s" \n' % (command
)
220 elif xsl_transform
== 'xalan':
221 command
= 'xalan.sh '
222 command
+= ' -OUT %s' % (out_file
)
223 command
+= ' -IN %s -XSL %s' % (xml_file
, xsl_file
)
224 params
= param_dict
.keys()
226 command
+= ' -PARAM %s "%s" ' % (param
, param_dict
[param
])
227 status
, output
= commands
.getstatusoutput(command
)
229 msg
= 'error converting %s to Fo\n' % (xml_file
)
230 msg
+= 'command = "%s" \n' % (command
)
234 elif xsl_transform
== 'lxml':
235 error
= transform_lxml(xslt_file
= xsl_file
, xml_file
= xml_file
, param_dict
=param_dict
, out_file
= out_file
)
237 error_func(error
, xml_file
)
239 sys
.stderr
.write('"%s" not a valid xsl_transform. Fix. Script now quitting\n' % (xsl_transform
))
243 def convert_to_xml(path_list
):
245 sys
.stderr
.write('converting to xml...\n')
246 num_files
= len(path_list
)
248 for the_path
in path_list
:
250 base
, ext
= os
.path
.splitext(the_path
)
251 out_file
= '%s.xml' % (base
)
253 sys
.stderr
.write('converting "%s" to "%s"\n' % (the_path
, out_file
))
254 publish_xml_cmdline (in_path
= the_path
, out_path
= out_file
)
256 # simple, fail-proof method
257 def convert_to_xml_command(path_list
):
258 # print 'converting to xml...'
259 num_files
= len(path_list
)
261 for the_path
in path_list
:
263 # print 'converting %s of %s files' % (counter, num_files)
264 base
, ext
= os
.path
.splitext(the_path
)
265 out_file
= '%s.xml' % (base
)
266 command
= 'rst2xml.py --trim-footnote-reference-space %s %s' % (the_path
, out_file
)
267 status
, output
= commands
.getstatusoutput(command
)
269 def convert_to_fo(xsl_transform
):
271 sys
.stderr
.write('converting to fo...\n')
272 xml_files
= glob
.glob('*.xml')
273 len_simple
= len(xml_files
)
274 the_keys
= test_dict
.keys()
275 for the_key
in the_keys
:
276 if the_key
not in xml_files
:
277 sys
.stderr
.write('%s not found in test_files; fix. Script now quitting' % (the_key
))
279 len_complex
= len(the_keys
)
281 for the_key
in the_keys
:
282 this_inside
= len(test_dict
[the_key
])
283 len_inside
+= this_inside
284 num_files
= len_simple
- len_complex
+ len_inside
286 for xml_file
in xml_files
:
287 params
= {'strict':'True'}
288 transform_info
= test_dict
.get(xml_file
)
290 for info
in transform_info
:
293 sys
.stderr
.write('converting "%s" to %s to FO\n' % (xml_file
, out_file
))
294 # print 'converting %s of %s files' % (counter, num_files)
295 added_params
= info
[0]
297 params
.update(added_params
)
298 transform_xsl(xsl_file
= '../docutilsToFo/xsl_fo/docutils_to_fo.xsl',
299 xml_file
= xml_file
, param_dict
= params
, out_file
= out_file
,
300 xsl_transform
= xsl_transform
)
302 validate_the_fo(out_file
)
306 # print 'converting %s of %s files' % (counter, num_files)
307 base
, ext
= os
.path
.splitext(xml_file
)
308 out_file
= '%s.fo' % (base
)
310 sys
.stderr
.write('converting "%s" to %s FO\n' % (xml_file
, out_file
))
311 transform_xsl(xsl_file
= '../docutilsToFo/xsl_fo/docutils_to_fo.xsl', xml_file
= xml_file
, param_dict
=params
, xsl_transform
= xsl_transform
)
312 validate_the_fo(out_file
)
314 def convert_to_pdf():
315 # print 'converting to pdf...'
316 fo_files
= glob
.glob('*.fo')
317 num_files
= len(fo_files
)
319 for fo_file
in fo_files
:
321 # print 'converting %s of %s files' % (counter, num_files)
322 base
, ext
= os
.path
.splitext(fo_file
)
323 out_file
= '%s.pdf' % (base
)
324 command
= 'fop %s %s' % (fo_file
, out_file
)
325 status
, output
= commands
.getstatusoutput(command
)
327 msg
= 'error converting %s to PDF\n' % (fo_file
)
328 msg
+= 'command = "%s" \n' % (command
)
333 def convert_fo_to_pdf(fo_file
, out_file
):
334 command
= 'fop %s %s' % (fo_file
, out_file
)
336 sys
.stderr
.write('\n doing command "%s"\n' % (command
))
337 status
, output
= commands
.getstatusoutput(command
)
339 msg
= 'error converting %s to PDF\n' % (fo_file
)
340 msg
+= 'command = "%s" \n' % (command
)
347 def make_conf_file(the_dict
, section
= 'FO', conf_path
= 'docutils.conf'):
349 write_obj
= file(conf_path
, 'w')
350 the_keys
= the_dict
.keys()
351 write_obj
.write('[%s]\n' % section
)
352 for the_key
in the_keys
:
353 write_obj
.write('%s = %s\n' % (the_key
, the_dict
[the_key
]))
357 def run_docutils_command(command
):
358 status
, output
= commands
.getstatusoutput(command
)
360 msg
= 'error converting to FO with docutilst_to_fo.py:\n'
361 msg
+= 'command = "%s" \n' % (command
)
366 def validate_the_fo(xml_file
):
367 error
= validate_fo_xsl(xml_file
)
369 sys
.stderr
.write('Problems converting "%s"\n' % (xml_file
))
372 def test_docutils_to_fo_script(script_command
):
374 sys
.stderr
.write('\n\n==========================================\n')
375 sys
.stderr
.write('TESTING DOCUTILS SCRIPT\n')
376 sys
.stderr
.write('\n==========================================\n')
377 command
= '%s -h' % script_command
378 status
, output
= commands
.getstatusoutput(command
)
379 default__commands
= ' --config docutils.conf '
380 for info
in docfo_commands
:
381 command
= script_command
383 filename
,ext
= os
.path
.splitext(in_file
)
384 out_file
= '%s.fo' % filename
385 out_opt
= ' -o %s ' % out_file
386 config_file
= os
.path
.join('conf_files', info
[1])
387 if not os
.path
.isfile(config_file
):
388 sys
.stderr
.write('Can\'t find "%s" conf file\n' % config_file
)
390 config_opt
= ' --config %s ' % config_file
391 command
+= ' %s %s %s' % (out_opt
, config_opt
, in_file
)
393 sys
.stderr
.write('Converting "%s" to "%s" with "%s" \n' % (in_file
, out_file
, config_file
))
394 run_docutils_command(command
)
395 validate_the_fo(out_file
)
397 out_pdf
= '%s.pdf' % filename
398 convert_fo_to_pdf(fo_file
= out_file
, out_file
= out_pdf
)
402 remove_paths(glob
.glob('*.xml'))
403 remove_paths(glob
.glob('*.xsl'))
404 remove_paths(glob
.glob('*.fo'))
405 remove_paths(glob
.glob('*.pdf'))
407 def rm_path(the_path
):
413 def remove_paths(list_of_paths
):
414 for the_path
in list_of_paths
:
420 sys
.stderr
.write('In FULL mode\n')
422 script_path
= os
.path
.abspath(os
.path
.join('scripts', 'docutils_to_fo.py'))
423 script_command
= 'python %s ' % script_path
424 current_dir
= os
.getcwd()
425 os
.chdir('test_files')
426 rst_files
= glob
.glob('*.rst')
427 convert_to_xml(rst_files
)
428 xsl_trans
= ['xsltproc', 'saxon', 'xalan', 'lxml',]
429 for xsl_tran
in xsl_trans
:
431 sys
.stderr
.write('Transforming to FO with %s\n' % (xsl_tran
))
432 convert_to_fo(xsl_transform
= xsl_tran
)
433 if xsl_tran
!= 'lxml':
434 remove_paths(glob
.glob('*.fo'))
436 test_docutils_to_fo_script(script_command
)
438 os
.chdir(current_dir
)
440 sys
.stderr
.write('\n\nSUCCESS!\n')
444 script_path
= os
.path
.abspath(os
.path
.join('scripts', 'docutils_to_fo.py'))
445 script_command
= 'python %s ' % script_path
446 current_dir
= os
.getcwd()
447 os
.chdir('test_files')
449 rst_files
= glob
.glob('*.rst')
450 convert_to_xml(rst_files
)
452 convert_to_fo(xsl_transform
= arg
.xsl_transform
[0])
455 for fo_file
in glob
.glob('*.fo'):
457 test_docutils_to_fo_script(script_command
)
461 os
.chdir(current_dir
)
465 locale
.setlocale(locale
.LC_ALL
, '')