scripts/lilypond-book.py

   1 #!@TARGET_PYTHON@
   2 # -*- coding: utf-8 -*-
   3
   4 # This file is part of LilyPond, the GNU music typesetter.
   5 #
   6 # LilyPond is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # LilyPond is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with LilyPond.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 '''
  20 Example usage:
  21
  22 test:
  23   lilypond-book --filter="tr '[a-z]' '[A-Z]'" BOOK
  24
  25 convert-ly on book:
  26   lilypond-book --filter="convert-ly --no-version --from=1.6.11 -" BOOK
  27
  28 classic lilypond-book:
  29   lilypond-book --process="lilypond" BOOK.tely
  30
  31 TODO:
  32
  33   *  ly-options: intertext?
  34   *  --line-width?
  35   *  eps in latex / eps by lilypond -b ps?
  36   *  check latex parameters, twocolumn, multicolumn?
  37   *  use --png --ps --pdf for making images?
  38
  39   *  Converting from lilypond-book source, substitute:
  40    @mbinclude foo.itely -> @include foo.itely
  41    \mbinput -> \input
  42
  43 '''
  44
  45
  46 # TODO: Better solve the global_options copying to the snippets...
  47
  48 import glob
  49 import os
  50 import re
  51 import stat
  52 import sys
  53 import tempfile
  54 import imp
  55 from optparse import OptionGroup
  56
  57
  58 """
  59 @relocate-preamble@
  60 """
  61
  62 import lilylib as ly
  63 import fontextract
  64 import langdefs
  65 global _;_=ly._
  66
  67 import book_base as BookBase
  68 import book_snippets as BookSnippet
  69 import book_html
  70 import book_docbook
  71 import book_texinfo
  72 import book_latex
  73
  74 ly.require_python_version ()
  75
  76 original_dir = os.getcwd ()
  77 backend = 'ps'
  78
  79 help_summary = (
  80 _ ("Process LilyPond snippets in hybrid HTML, LaTeX, texinfo or DocBook document.")
  81 + '\n\n'
  82 + _ ("Examples:")
  83 + '''
  84  $ lilypond-book --filter="tr '[a-z]' '[A-Z]'" %(BOOK)s
  85  $ lilypond-book -F "convert-ly --no-version --from=2.0.0 -" %(BOOK)s
  86  $ lilypond-book --process='lilypond -I include' %(BOOK)s
  87 ''' % {'BOOK': _ ("BOOK")})
  88
  89 authors = ('Jan Nieuwenhuizen <janneke@gnu.org>',
  90            'Han-Wen Nienhuys <hanwen@xs4all.nl>')
  91
  92 ################################################################
  93 def exit (i):
  94     if global_options.verbose:
  95         raise Exception (_ ('Exiting (%d)...') % i)
  96     else:
  97         sys.exit (i)
  98
  99 def identify ():
 100     ly.encoded_write (sys.stdout, '%s (GNU LilyPond) %s\n' % (ly.program_name, ly.program_version))
 101
 102 progress = ly.progress
 103 warning = ly.warning
 104 error = ly.error
 105
 106
 107 def warranty ():
 108     identify ()
 109     ly.encoded_write (sys.stdout, '''
 110 %s
 111
 112   %s
 113
 114 %s
 115 %s
 116 ''' % ( _ ('Copyright (c) %s by') % '2001--2011',
 117         '\n  '.join (authors),
 118         _ ("Distributed under terms of the GNU General Public License."),
 119         _ ("It comes with NO WARRANTY.")))
 120
 121 def get_option_parser ():
 122     p = ly.get_option_parser (usage=_ ("%s [OPTION]... FILE") % 'lilypond-book',
 123                               description=help_summary,
 124                               conflict_handler="resolve",
 125                               add_help_option=False)
 126
 127     p.add_option ('-F', '--filter', metavar=_ ("FILTER"),
 128                   action="store",
 129                   dest="filter_cmd",
 130                   help=_ ("pipe snippets through FILTER [default: `convert-ly -n -']"),
 131                   default=None)
 132
 133     p.add_option ('-f', '--format',
 134                   help=_ ("use output format FORMAT (texi [default], texi-html, latex, html, docbook)"),
 135                   metavar=_ ("FORMAT"),
 136                   action='store')
 137
 138     p.add_option("-h", "--help",
 139                  action="help",
 140                  help=_ ("show this help and exit"))
 141
 142     p.add_option ("-I", '--include', help=_ ("add DIR to include path"),
 143                   metavar=_ ("DIR"),
 144                   action='append', dest='include_path',
 145                   default=[os.path.abspath (os.getcwd ())])
 146
 147     p.add_option ('--info-images-dir',
 148                   help=_ ("format Texinfo output so that Info will "
 149                           "look for images of music in DIR"),
 150                   metavar=_ ("DIR"),
 151                   action='store', dest='info_images_dir',
 152                   default='')
 153
 154     p.add_option ('--left-padding',
 155                   metavar=_ ("PAD"),
 156                   dest="padding_mm",
 157                   help=_ ("pad left side of music to align music inspite of uneven bar numbers (in mm)"),
 158                   type="float",
 159                   default=3.0)
 160
 161     p.add_option ('--lily-output-dir',
 162                   help=_ ("write lily-XXX files to DIR, link into --output dir"),
 163                   metavar=_ ("DIR"),
 164                   action='store', dest='lily_output_dir',
 165                   default=None)
 166
 167     p.add_option ('--load-custom-package', help=_ ("Load the additional python PACKAGE (containing e.g. a custom output format)"),
 168                   metavar=_ ("PACKAGE"),
 169                   action='append', dest='custom_packages',
 170                   default=[])
 171
 172     p.add_option ("-o", '--output', help=_ ("write output to DIR"),
 173                   metavar=_ ("DIR"),
 174                   action='store', dest='output_dir',
 175                   default='')
 176
 177     p.add_option ('-P', '--process', metavar=_ ("COMMAND"),
 178                   help = _ ("process ly_files using COMMAND FILE..."),
 179                   action='store',
 180                   dest='process_cmd', default='')
 181
 182     p.add_option ('-s', '--safe', help=_ ("Compile snippets in safe mode"),
 183                   action="store_true",
 184                   default=False,
 185                   dest="safe_mode")
 186
 187     p.add_option ('--skip-lily-check',
 188                   help=_ ("do not fail if no lilypond output is found"),
 189                   metavar=_ ("DIR"),
 190                   action='store_true', dest='skip_lilypond_run',
 191                   default=False)
 192
 193     p.add_option ('--skip-png-check',
 194                   help=_ ("do not fail if no PNG images are found for EPS files"),
 195                   metavar=_ ("DIR"),
 196                   action='store_true', dest='skip_png_check',
 197                   default=False)
 198
 199     p.add_option ('--use-source-file-names',
 200                   help=_ ("write snippet output files with the same base name as their source file"),
 201                   action='store_true', dest='use_source_file_names',
 202                   default=False)
 203
 204     p.add_option ('-V', '--verbose', help=_ ("be verbose"),
 205                   action="store_true",
 206                   default=False,
 207                   dest="verbose")
 208
 209     p.version = "@TOPLEVEL_VERSION@"
 210     p.add_option("--version",
 211                  action="version",
 212                  help=_ ("show version number and exit"))
 213
 214     p.add_option ('-w', '--warranty',
 215                   help=_ ("show warranty and copyright"),
 216                   action='store_true')
 217
 218     group = OptionGroup (p, "Options only for the latex and texinfo backends")
 219     group.add_option ('--latex-program',
 220               help=_ ("run executable PROG instead of latex, or in\n\
 221 case --pdf option is set instead of pdflatex"),
 222               metavar=_ ("PROG"),
 223               action='store', dest='latex_program',
 224               default='latex')
 225     group.add_option ('--pdf',
 226               action="store_true",
 227               dest="create_pdf",
 228               help=_ ("create PDF files for use with PDFTeX"),
 229               default=False)
 230     p.add_option_group (group)
 231
 232     p.add_option_group ('',
 233                         description=(
 234         _ ("Report bugs via %s")
 235         % ' http://post.gmane.org/post.php'
 236         '?group=gmane.comp.gnu.lilypond.bugs') + '\n')
 237
 238
 239     for formatter in BookBase.all_formats:
 240       formatter.add_options (p)
 241
 242     return p
 243
 244 lilypond_binary = os.path.join ('@bindir@', 'lilypond')
 245
 246 # If we are called with full path, try to use lilypond binary
 247 # installed in the same path; this is needed in GUB binaries, where
 248 # @bindir is always different from the installed binary path.
 249 if 'bindir' in globals () and bindir:
 250     lilypond_binary = os.path.join (bindir, 'lilypond')
 251
 252 # Only use installed binary when we are installed too.
 253 if '@bindir@' == ('@' + 'bindir@') or not os.path.exists (lilypond_binary):
 254     lilypond_binary = 'lilypond'
 255
 256 global_options = None
 257
 258
 259
 260
 261 def find_linestarts (s):
 262     nls = [0]
 263     start = 0
 264     end = len (s)
 265     while 1:
 266         i = s.find ('\n', start)
 267         if i < 0:
 268             break
 269
 270         i = i + 1
 271         nls.append (i)
 272         start = i
 273
 274     nls.append (len (s))
 275     return nls
 276
 277 def find_toplevel_snippets (input_string, formatter):
 278     res = {}
 279     types = formatter.supported_snippet_types ()
 280     for t in types:
 281         res[t] = re.compile (formatter.snippet_regexp (t))
 282
 283     snippets = []
 284     index = 0
 285     found = dict ([(t, None) for t in types])
 286
 287     line_starts = find_linestarts (input_string)
 288     line_start_idx = 0
 289     # We want to search for multiple regexes, without searching
 290     # the string multiple times for one regex.
 291     # Hence, we use earlier results to limit the string portion
 292     # where we search.
 293     # Since every part of the string is traversed at most once for
 294     # every type of snippet, this is linear.
 295     while 1:
 296         first = None
 297         endex = 1 << 30
 298         for type in types:
 299             if not found[type] or found[type][0] < index:
 300                 found[type] = None
 301
 302                 m = res[type].search (input_string[index:endex])
 303                 if not m:
 304                     continue
 305
 306                 klass = global_options.formatter.snippet_class (type)
 307
 308                 start = index + m.start ('match')
 309                 line_number = line_start_idx
 310                 while (line_starts[line_number] < start):
 311                     line_number += 1
 312
 313                 line_number += 1
 314                 snip = klass (type, m, formatter, line_number, global_options)
 315
 316                 found[type] = (start, snip)
 317
 318             if (found[type]
 319                 and (not first
 320                      or found[type][0] < found[first][0])):
 321                 first = type
 322
 323                 # FIXME.
 324
 325                 # Limiting the search space is a cute
 326                 # idea, but this *requires* to search
 327                 # for possible containing blocks
 328                 # first, at least as long as we do not
 329                 # search for the start of blocks, but
 330                 # always/directly for the entire
 331                 # @block ... @end block.
 332
 333                 endex = found[first][0]
 334
 335         if not first:
 336             snippets.append (BookSnippet.Substring (input_string, index, len (input_string), line_start_idx))
 337             break
 338
 339         while (start > line_starts[line_start_idx+1]):
 340             line_start_idx += 1
 341
 342         (start, snip) = found[first]
 343         snippets.append (BookSnippet.Substring (input_string, index, start, line_start_idx + 1))
 344         snippets.append (snip)
 345         found[first] = None
 346         index = start + len (snip.match.group ('match'))
 347
 348     return snippets
 349
 350 def system_in_directory (cmd, directory):
 351     """Execute a command in a different directory.
 352
 353     Because of win32 compatibility, we can't simply use subprocess.
 354     """
 355
 356     current = os.getcwd()
 357     os.chdir (directory)
 358     ly.system(cmd, be_verbose=global_options.verbose,
 359               progress_p=1)
 360     os.chdir (current)
 361
 362
 363 def process_snippets (cmd, snippets,
 364                       formatter, lily_output_dir):
 365     """Run cmd on all of the .ly files from snippets."""
 366
 367     if not snippets:
 368         return
 369
 370     cmd = formatter.adjust_snippet_command (cmd)
 371
 372     checksum = snippet_list_checksum (snippets)
 373     contents = '\n'.join (['snippet-map-%d.ly' % checksum]
 374                           + list (set ([snip.basename() + '.ly' for snip in snippets])))
 375     name = os.path.join (lily_output_dir,
 376                          'snippet-names-%d.ly' % checksum)
 377     file (name, 'wb').write (contents)
 378
 379     system_in_directory (' '.join ([cmd, ly.mkarg (name)]),
 380                          lily_output_dir)
 381
 382
 383 def snippet_list_checksum (snippets):
 384     return hash (' '.join([l.basename() for l in snippets]))
 385
 386 def write_file_map (lys, name):
 387     snippet_map = file (os.path.join (
 388         global_options.lily_output_dir,
 389         'snippet-map-%d.ly' % snippet_list_checksum (lys)), 'w')
 390
 391     snippet_map.write ("""
 392 #(define version-seen #t)
 393 #(define output-empty-score-list #f)
 394 #(ly:add-file-name-alist '(%s
 395     ))\n
 396 """ % '\n'.join(['("%s.ly" . "%s")\n' % (ly.basename (), name)
 397                  for ly in lys]))
 398
 399 def split_output_files(directory):
 400     """Returns directory entries in DIRECTORY/XX/ , where XX are hex digits.
 401
 402     Return value is a set of strings.
 403     """
 404     files = []
 405     for subdir in glob.glob (os.path.join (directory, '[a-f0-9][a-f0-9]')):
 406         base_subdir = os.path.split (subdir)[1]
 407         sub_files = [os.path.join (base_subdir, name)
 408                      for name in os.listdir (subdir)]
 409         files += sub_files
 410     return set (files)
 411
 412 def do_process_cmd (chunks, input_name, options):
 413     snippets = [c for c in chunks if isinstance (c, BookSnippet.LilypondSnippet)]
 414
 415     output_files = split_output_files (options.lily_output_dir)
 416     outdated = [c for c in snippets if c.is_outdated (options.lily_output_dir, output_files)]
 417
 418     write_file_map (outdated, input_name)
 419     progress (_ ("Writing snippets..."))
 420     for snippet in outdated:
 421         snippet.write_ly()
 422     progress ('\n')
 423
 424     if outdated:
 425         progress (_ ("Processing..."))
 426         progress ('\n')
 427         process_snippets (options.process_cmd, outdated,
 428                           options.formatter, options.lily_output_dir)
 429
 430     else:
 431         progress (_ ("All snippets are up to date..."))
 432
 433     if options.lily_output_dir != options.output_dir:
 434         output_files = split_output_files (options.lily_output_dir)
 435         for snippet in snippets:
 436             snippet.link_all_output_files (options.lily_output_dir,
 437                                            output_files,
 438                                            options.output_dir)
 439
 440     progress ('\n')
 441
 442
 443 ###
 444 # Format guessing data
 445
 446 def guess_format (input_filename):
 447     format = None
 448     e = os.path.splitext (input_filename)[1]
 449     for formatter in BookBase.all_formats:
 450       if formatter.can_handle_extension (e):
 451         return formatter
 452     error (_ ("cannot determine format for: %s" % input_filename))
 453     exit (1)
 454
 455 def write_if_updated (file_name, lines):
 456     try:
 457         f = file (file_name)
 458         oldstr = f.read ()
 459         new_str = ''.join (lines)
 460         if oldstr == new_str:
 461             progress (_ ("%s is up to date.") % file_name)
 462             progress ('\n')
 463
 464             # this prevents make from always rerunning lilypond-book:
 465             # output file must be touched in order to be up to date
 466             os.utime (file_name, None)
 467             return
 468     except:
 469         pass
 470
 471     output_dir = os.path.dirname (file_name)
 472     if not os.path.exists (output_dir):
 473         os.makedirs (output_dir)
 474
 475     progress (_ ("Writing `%s'...") % file_name)
 476     file (file_name, 'w').writelines (lines)
 477     progress ('\n')
 478
 479
 480 def note_input_file (name, inputs=[]):
 481     ## hack: inputs is mutable!
 482     inputs.append (name)
 483     return inputs
 484
 485 def samefile (f1, f2):
 486     try:
 487         return os.path.samefile (f1, f2)
 488     except AttributeError:                # Windoze
 489         f1 = re.sub ("//*", "/", f1)
 490         f2 = re.sub ("//*", "/", f2)
 491         return f1 == f2
 492
 493 def do_file (input_filename, included=False):
 494     # Ugh.
 495     input_absname = input_filename
 496     if not input_filename or input_filename == '-':
 497         in_handle = sys.stdin
 498         input_fullname = '<stdin>'
 499     else:
 500         if os.path.exists (input_filename):
 501             input_fullname = input_filename
 502         else:
 503             input_fullname = global_options.formatter.input_fullname (input_filename)
 504         # Normalize path to absolute path, since we will change cwd to the output dir!
 505         # Otherwise, "lilypond-book -o out test.tex" will complain that it is
 506         # overwriting the input file (which it is actually not), since the
 507         # input filename is relative to the CWD...
 508         input_absname = os.path.abspath (input_fullname)
 509
 510         note_input_file (input_fullname)
 511         in_handle = file (input_fullname)
 512
 513     if input_filename == '-':
 514         input_base = 'stdin'
 515     elif included:
 516         input_base = os.path.splitext (input_filename)[0]
 517     else:
 518         input_base = os.path.basename (
 519             os.path.splitext (input_filename)[0])
 520
 521     # don't complain when global_options.output_dir is existing
 522     if not global_options.output_dir:
 523         global_options.output_dir = os.getcwd()
 524     else:
 525         global_options.output_dir = os.path.abspath(global_options.output_dir)
 526
 527         if not os.path.isdir (global_options.output_dir):
 528             os.mkdir (global_options.output_dir, 0777)
 529         os.chdir (global_options.output_dir)
 530
 531     output_filename = os.path.join(global_options.output_dir,
 532                                    input_base + global_options.formatter.default_extension)
 533     if (os.path.exists (input_filename)
 534         and os.path.exists (output_filename)
 535         and samefile (output_filename, input_absname)):
 536      error (
 537      _ ("Output would overwrite input file; use --output."))
 538      exit (2)
 539
 540     try:
 541         progress (_ ("Reading %s...") % input_fullname)
 542         source = in_handle.read ()
 543         progress ('\n')
 544
 545         if not included:
 546             global_options.formatter.init_default_snippet_options (source)
 547
 548
 549         progress (_ ("Dissecting..."))
 550         chunks = find_toplevel_snippets (source, global_options.formatter)
 551
 552         # Let the formatter modify the chunks before further processing
 553         chunks = global_options.formatter.process_chunks (chunks)
 554         progress ('\n')
 555
 556         if global_options.filter_cmd:
 557             write_if_updated (output_filename,
 558                      [c.filter_text () for c in chunks])
 559         elif global_options.process_cmd:
 560             do_process_cmd (chunks, input_fullname, global_options)
 561             progress (_ ("Compiling %s...") % output_filename)
 562             progress ('\n')
 563             write_if_updated (output_filename,
 564                      [s.replacement_text ()
 565                      for s in chunks])
 566
 567         def process_include (snippet):
 568             os.chdir (original_dir)
 569             name = snippet.substring ('filename')
 570             progress (_ ("Processing include: %s") % name)
 571             progress ('\n')
 572             return do_file (name, included=True)
 573
 574         include_chunks = map (process_include,
 575                               filter (lambda x: isinstance (x, BookSnippet.IncludeSnippet),
 576                                       chunks))
 577
 578         return chunks + reduce (lambda x, y: x + y, include_chunks, [])
 579
 580     except BookSnippet.CompileError:
 581         os.chdir (original_dir)
 582         progress (_ ("Removing `%s'") % output_filename)
 583         progress ('\n')
 584         raise BookSnippet.CompileError
 585
 586 def do_options ():
 587     global global_options
 588
 589     opt_parser = get_option_parser()
 590     (global_options, args) = opt_parser.parse_args ()
 591
 592     global_options.information = {'program_version': ly.program_version, 'program_name': ly.program_name }
 593
 594     global_options.include_path =  map (os.path.abspath, global_options.include_path)
 595
 596     # Load the python packages (containing e.g. custom formatter classes)
 597     # passed on the command line
 598     nr = 0
 599     for i in global_options.custom_packages:
 600         nr += 1
 601         print imp.load_source ("book_custom_package%s" % nr, i)
 602
 603
 604     if global_options.warranty:
 605         warranty ()
 606         exit (0)
 607     if not args or len (args) > 1:
 608         opt_parser.print_help ()
 609         exit (2)
 610
 611     return args
 612
 613 def main ():
 614     # FIXME: 85 lines of `main' macramee??
 615     files = do_options ()
 616
 617     basename = os.path.splitext (files[0])[0]
 618     basename = os.path.split (basename)[1]
 619
 620     if global_options.format:
 621       # Retrieve the formatter for the given format
 622       for formatter in BookBase.all_formats:
 623         if formatter.can_handle_format (global_options.format):
 624           global_options.formatter = formatter
 625     else:
 626         global_options.formatter = guess_format (files[0])
 627         global_options.format = global_options.formatter.format
 628
 629     # make the global options available to the formatters:
 630     global_options.formatter.global_options = global_options
 631     formats = global_options.formatter.image_formats
 632
 633     if global_options.process_cmd == '':
 634         global_options.process_cmd = (lilypond_binary
 635                                       + ' --formats=%s -dbackend=eps ' % formats)
 636
 637     if global_options.process_cmd:
 638         includes = global_options.include_path
 639         if global_options.lily_output_dir:
 640             # This must be first, so lilypond prefers to read .ly
 641             # files in the other lybookdb dir.
 642             includes = [os.path.abspath(global_options.lily_output_dir)] + includes
 643         global_options.process_cmd += ' '.join ([' -I %s' % ly.mkarg (p)
 644                                                  for p in includes])
 645
 646     global_options.formatter.process_options (global_options)
 647
 648     if global_options.verbose:
 649         global_options.process_cmd += " --verbose "
 650
 651     if global_options.padding_mm:
 652         global_options.process_cmd += " -deps-box-padding=%f " % global_options.padding_mm
 653
 654     global_options.process_cmd += " -dread-file-list -dno-strip-output-dir"
 655
 656     if global_options.lily_output_dir:
 657         global_options.lily_output_dir = os.path.abspath(global_options.lily_output_dir)
 658         if not os.path.isdir (global_options.lily_output_dir):
 659             os.makedirs (global_options.lily_output_dir)
 660     else:
 661         global_options.lily_output_dir = os.path.abspath(global_options.output_dir)
 662
 663
 664     identify ()
 665     try:
 666         chunks = do_file (files[0])
 667     except BookSnippet.CompileError:
 668         exit (1)
 669
 670     inputs = note_input_file ('')
 671     inputs.pop ()
 672
 673     base_file_name = os.path.splitext (os.path.basename (files[0]))[0]
 674     dep_file = os.path.join (global_options.output_dir, base_file_name + '.dep')
 675     final_output_file = os.path.join (global_options.output_dir,
 676                      base_file_name
 677                      + '.%s' % global_options.format)
 678
 679     os.chdir (original_dir)
 680     file (dep_file, 'w').write ('%s: %s'
 681                                 % (final_output_file, ' '.join (inputs)))
 682
 683 if __name__ == '__main__':
 684     main ()