plugins/video/transcode.py

   1 import logging
   2 import math
   3 import os
   4 import re
   5 import shutil
   6 import subprocess
   7 import sys
   8 import tempfile
   9 import threading
  10 import time
  11
  12 import lrucache
  13
  14 import config
  15 import metadata
  16
  17 logger = logging.getLogger('pyTivo.video.transcode')
  18
  19 info_cache = lrucache.LRUCache(1000)
  20 ffmpeg_procs = {}
  21 reapers = {}
  22
  23 GOOD_MPEG_FPS = ['23.98', '24.00', '25.00', '29.97',
  24                  '30.00', '50.00', '59.94', '60.00']
  25
  26 BLOCKSIZE = 512 * 1024
  27 MAXBLOCKS = 2
  28 TIMEOUT = 600
  29
  30 UNSET = 0
  31 OLD_PAD = 1
  32 NEW_PAD = 2
  33
  34 pad_style = UNSET
  35
  36 # XXX BIG HACK
  37 # subprocess is broken for me on windows so super hack
  38 def patchSubprocess():
  39     o = subprocess.Popen._make_inheritable
  40
  41     def _make_inheritable(self, handle):
  42         if not handle: return subprocess.GetCurrentProcess()
  43         return o(self, handle)
  44
  45     subprocess.Popen._make_inheritable = _make_inheritable
  46 mswindows = (sys.platform == "win32")
  47 if mswindows:
  48     patchSubprocess()
  49
  50 def debug(msg):
  51     if type(msg) == str:
  52         try:
  53             msg = msg.decode('utf8')
  54         except:
  55             if sys.platform == 'darwin':
  56                 msg = msg.decode('macroman')
  57             else:
  58                 msg = msg.decode('iso8859-1')
  59     logger.debug(msg)
  60
  61 def transcode(isQuery, inFile, outFile, tsn='', mime='', thead=''):
  62     settings = {'video_codec': select_videocodec(inFile, tsn, mime),
  63                 'video_br': select_videobr(inFile, tsn),
  64                 'video_fps': select_videofps(inFile, tsn),
  65                 'max_video_br': select_maxvideobr(tsn),
  66                 'buff_size': select_buffsize(tsn),
  67                 'aspect_ratio': ' '.join(select_aspect(inFile, tsn)),
  68                 'audio_br': select_audiobr(tsn),
  69                 'audio_fr': select_audiofr(inFile, tsn),
  70                 'audio_ch': select_audioch(inFile, tsn),
  71                 'audio_codec': select_audiocodec(isQuery, inFile, tsn),
  72                 'audio_lang': select_audiolang(inFile, tsn),
  73                 'ffmpeg_pram': select_ffmpegprams(tsn),
  74                 'format': select_format(tsn, mime)}
  75
  76     if isQuery:
  77         return settings
  78
  79     ffmpeg_path = config.get_bin('ffmpeg')
  80     cmd_string = config.getFFmpegTemplate(tsn) % settings
  81     fname = unicode(inFile, 'utf-8')
  82     if mswindows:
  83         fname = fname.encode('iso8859-1')
  84
  85     if inFile[-5:].lower() == '.tivo':
  86         tivodecode_path = config.get_bin('tivodecode')
  87         tivo_mak = config.get_server('tivo_mak')
  88         tcmd = [tivodecode_path, '-m', tivo_mak, fname]
  89         tivodecode = subprocess.Popen(tcmd, stdout=subprocess.PIPE,
  90                                       bufsize=(512 * 1024))
  91         if tivo_compatible(inFile, tsn)[0]:
  92             cmd = ''
  93             ffmpeg = tivodecode
  94         else:
  95             cmd = [ffmpeg_path, '-i', '-'] + cmd_string.split()
  96             ffmpeg = subprocess.Popen(cmd, stdin=tivodecode.stdout,
  97                                       stdout=subprocess.PIPE,
  98                                       bufsize=(512 * 1024))
  99     else:
 100         cmd = [ffmpeg_path, '-i', fname] + cmd_string.split()
 101         ffmpeg = subprocess.Popen(cmd, bufsize=(512 * 1024),
 102                                   stdout=subprocess.PIPE)
 103
 104     if cmd:
 105         debug('transcoding to tivo model ' + tsn[:3] + ' using ffmpeg command:')
 106         debug(' '.join(cmd))
 107
 108     ffmpeg_procs[inFile] = {'process': ffmpeg, 'start': 0, 'end': 0,
 109                             'last_read': time.time(), 'blocks': []}
 110     if thead:
 111         ffmpeg_procs[inFile]['blocks'].append(thead)
 112     reap_process(inFile)
 113     return resume_transfer(inFile, outFile, 0)
 114
 115 def is_resumable(inFile, offset):
 116     if inFile in ffmpeg_procs:
 117         proc = ffmpeg_procs[inFile]
 118         if proc['start'] <= offset < proc['end']:
 119             return True
 120         else:
 121             cleanup(inFile)
 122             kill(proc['process'])
 123     return False
 124
 125 def resume_transfer(inFile, outFile, offset):
 126     proc = ffmpeg_procs[inFile]
 127     offset -= proc['start']
 128     count = 0
 129
 130     try:
 131         for block in proc['blocks']:
 132             length = len(block)
 133             if offset < length:
 134                 if offset > 0:
 135                     block = block[offset:]
 136                 outFile.write('%x\r\n' % len(block))
 137                 outFile.write(block)
 138                 outFile.write('\r\n')
 139                 count += len(block)
 140             offset -= length
 141         outFile.flush()
 142     except Exception, msg:
 143         logger.info(msg)
 144         return count
 145
 146     proc['start'] = proc['end']
 147     proc['blocks'] = []
 148
 149     return count + transfer_blocks(inFile, outFile)
 150
 151 def transfer_blocks(inFile, outFile):
 152     proc = ffmpeg_procs[inFile]
 153     blocks = proc['blocks']
 154     count = 0
 155
 156     while True:
 157         try:
 158             block = proc['process'].stdout.read(BLOCKSIZE)
 159             proc['last_read'] = time.time()
 160         except Exception, msg:
 161             logger.info(msg)
 162             cleanup(inFile)
 163             kill(proc['process'])
 164             break
 165
 166         if not block:
 167             try:
 168                 outFile.flush()
 169             except Exception, msg:
 170                 logger.info(msg)
 171             else:
 172                 cleanup(inFile)
 173             break
 174
 175         blocks.append(block)
 176         proc['end'] += len(block)
 177         if len(blocks) > MAXBLOCKS:
 178             proc['start'] += len(blocks[0])
 179             blocks.pop(0)
 180
 181         try:
 182             outFile.write('%x\r\n' % len(block))
 183             outFile.write(block)
 184             outFile.write('\r\n')
 185             count += len(block)
 186         except Exception, msg:
 187             logger.info(msg)
 188             break
 189
 190     return count
 191
 192 def reap_process(inFile):
 193     if ffmpeg_procs and inFile in ffmpeg_procs:
 194         proc = ffmpeg_procs[inFile]
 195         if proc['last_read'] + TIMEOUT < time.time():
 196             del ffmpeg_procs[inFile]
 197             del reapers[inFile]
 198             kill(proc['process'])
 199         else:
 200             reaper = threading.Timer(TIMEOUT, reap_process, (inFile,))
 201             reapers[inFile] = reaper
 202             reaper.start()
 203
 204 def cleanup(inFile):
 205     del ffmpeg_procs[inFile]
 206     reapers[inFile].cancel()
 207     del reapers[inFile]
 208
 209 def select_audiocodec(isQuery, inFile, tsn='', mime=''):
 210     if inFile[-5:].lower() == '.tivo':
 211         return '-acodec copy'
 212     vInfo = video_info(inFile)
 213     codectype = vInfo['vCodec']
 214     # Default, compatible with all TiVo's
 215     codec = 'ac3'
 216     if mime == 'video/mp4':
 217         compatiblecodecs = ('mpeg4aac', 'libfaad', 'mp4a', 'aac',
 218                             'ac3', 'liba52')
 219     else:
 220         compatiblecodecs = ('ac3', 'liba52', 'mp2')
 221
 222     if vInfo['aCodec'] in compatiblecodecs:
 223         aKbps = vInfo['aKbps']
 224         aCh = vInfo['aCh']
 225         if aKbps == None:
 226             if vInfo['aCodec'] in ('mpeg4aac', 'libfaad', 'mp4a', 'aac'):
 227                 # along with the channel check below this should
 228                 # pass any AAC audio that has undefined 'aKbps' and
 229                 # is <= 2 channels.  Should be TiVo compatible.
 230                 codec = 'copy'
 231             elif not isQuery:
 232                 vInfoQuery = audio_check(inFile, tsn)
 233                 if vInfoQuery == None:
 234                     aKbps = None
 235                     aCh = None
 236                 else:
 237                     aKbps = vInfoQuery['aKbps']
 238                     aCh = vInfoQuery['aCh']
 239             else:
 240                 codec = 'TBA'
 241         if aKbps and int(aKbps) <= config.getMaxAudioBR(tsn):
 242             # compatible codec and bitrate, do not reencode audio
 243             codec = 'copy'
 244         if vInfo['aCodec'] != 'ac3' and (aCh == None or aCh > 2):
 245             codec = 'ac3'
 246     copy_flag = config.get_tsn('copy_ts', tsn)
 247     copyts = ' -copyts'
 248     if ((codec == 'copy' and codectype == 'mpeg2video' and not copy_flag) or
 249         (copy_flag and copy_flag.lower() == 'false')):
 250         copyts = ''
 251     return '-acodec ' + codec + copyts
 252
 253 def select_audiofr(inFile, tsn):
 254     freq = '48000'  # default
 255     vInfo = video_info(inFile)
 256     if vInfo['aFreq'] == '44100':
 257         # compatible frequency
 258         freq = vInfo['aFreq']
 259     audio_fr = config.get_tsn('audio_fr', tsn)
 260     if audio_fr != None:
 261         freq = audio_fr
 262     return '-ar ' + freq
 263
 264 def select_audioch(inFile, tsn):
 265     ch = config.get_tsn('audio_ch', tsn)
 266     if ch:
 267         return '-ac ' + ch
 268     # AC-3 max channels is 5.1
 269     if video_info(inFile)['aCh'] > 6:
 270         debug('Too many audio channels for AC-3, using 5.1 instead')
 271         return '-ac 6'
 272     return ''
 273
 274 def select_audiolang(inFile, tsn):
 275     vInfo = video_info(inFile)
 276     audio_lang = config.get_tsn('audio_lang', tsn)
 277     debug('audio_lang: %s' % audio_lang)
 278     if vInfo['mapAudio']:
 279         # default to first detected audio stream to begin with
 280         stream = vInfo['mapAudio'][0][0]
 281     if audio_lang != None and vInfo['mapVideo'] != None:
 282         langmatch_curr = []
 283         langmatch_prev = vInfo['mapAudio'][:]
 284         for lang in audio_lang.replace(' ', '').lower().split(','):
 285             for s, l in langmatch_prev:
 286                 if lang in s + l.replace(' ', '').lower():
 287                     langmatch_curr.append((s, l))
 288                     stream = s
 289             # if only 1 item matched we're done
 290             if len(langmatch_curr) == 1:
 291                 break
 292             # if more than 1 item matched copy the curr area to the prev
 293             # array we only need to look at the new shorter list from
 294             # now on
 295             elif len(langmatch_curr) > 1:
 296                 langmatch_prev = langmatch_curr[:]
 297                 langmatch_curr = []
 298         # if we drop out of the loop with more than 1 item default to
 299         # the first item
 300         if len(langmatch_curr) > 1:
 301             stream = langmatch_curr[0][0]
 302     # don't let FFmpeg auto select audio stream, pyTivo defaults to
 303     # first detected
 304     if stream:
 305         debug('selected audio stream: %s' % stream)
 306         return '-map ' + vInfo['mapVideo'] + ' -map ' + stream
 307     # if no audio is found
 308     debug('selected audio stream: None detected')
 309     return ''
 310
 311 def select_videofps(inFile, tsn):
 312     vInfo = video_info(inFile)
 313     fps = '-r 29.97'  # default
 314     if config.isHDtivo(tsn) and vInfo['vFps'] in GOOD_MPEG_FPS:
 315         fps = ' '
 316     video_fps = config.get_tsn('video_fps', tsn)
 317     if video_fps != None:
 318         fps = '-r ' + video_fps
 319     return fps
 320
 321 def select_videocodec(inFile, tsn, mime=''):
 322     vInfo = video_info(inFile)
 323     if tivo_compatible_video(vInfo, tsn, mime)[0]:
 324         codec = 'copy'
 325         if (mime == 'video/x-tivo-mpeg-ts' and
 326             vInfo.get('vCodec', '') == 'h264'):
 327             codec += ' -bsf h264_mp4toannexb'
 328     else:
 329         codec = 'mpeg2video'  # default
 330     return '-vcodec ' + codec
 331
 332 def select_videobr(inFile, tsn, mime=''):
 333     return '-b ' + str(select_videostr(inFile, tsn, mime) / 1000) + 'k'
 334
 335 def select_videostr(inFile, tsn, mime=''):
 336     vInfo = video_info(inFile)
 337     if tivo_compatible_video(vInfo, tsn, mime)[0]:
 338         video_str = int(vInfo['kbps'])
 339         if vInfo['aKbps']:
 340             video_str -= int(vInfo['aKbps'])
 341         video_str *= 1000
 342     else:
 343         video_str = config.strtod(config.getVideoBR(tsn))
 344         if config.isHDtivo(tsn) and vInfo['kbps']:
 345             video_str = max(video_str, int(vInfo['kbps']) * 1000)
 346         video_str = int(min(config.strtod(config.getMaxVideoBR(tsn)) * 0.95,
 347                             video_str))
 348     return video_str
 349
 350 def select_audiobr(tsn):
 351     return '-ab ' + config.getAudioBR(tsn)
 352
 353 def select_maxvideobr(tsn):
 354     return '-maxrate ' + config.getMaxVideoBR(tsn)
 355
 356 def select_buffsize(tsn):
 357     return '-bufsize ' + config.getBuffSize(tsn)
 358
 359 def select_ffmpegprams(tsn):
 360     params = config.getFFmpegPrams(tsn)
 361     if not params:
 362         params = ''
 363     return params
 364
 365 def select_format(tsn, mime):
 366     if mime == 'video/x-tivo-mpeg-ts':
 367         fmt = 'mpegts'
 368     else:
 369         fmt = 'vob'
 370     return '-f %s -' % fmt
 371
 372 def pad_check():
 373     global pad_style
 374     if pad_style == UNSET:
 375         pad_style = OLD_PAD
 376         filters = tempfile.TemporaryFile()
 377         cmd = [config.get_bin('ffmpeg'), '-filters']
 378         ffmpeg = subprocess.Popen(cmd, stdout=filters, stderr=subprocess.PIPE)
 379         ffmpeg.wait()
 380         filters.seek(0)
 381         for line in filters:
 382             if line.startswith('pad'):
 383                 pad_style = NEW_PAD
 384                 break
 385         filters.close()
 386     return pad_style == NEW_PAD
 387
 388 def pad_TB(TIVO_WIDTH, TIVO_HEIGHT, multiplier, vInfo):
 389     endHeight = int(((TIVO_WIDTH * vInfo['vHeight']) /
 390                       vInfo['vWidth']) * multiplier)
 391     if endHeight % 2:
 392         endHeight -= 1
 393     topPadding = (TIVO_HEIGHT - endHeight) / 2
 394     if topPadding % 2:
 395         topPadding -= 1
 396     newpad = pad_check()
 397     if newpad:
 398         return ['-vf', 'scale=%d:%d,pad=%d:%d:0:%d' % (TIVO_WIDTH,
 399                 endHeight, TIVO_WIDTH, TIVO_HEIGHT, topPadding)]
 400     else:
 401         bottomPadding = (TIVO_HEIGHT - endHeight) - topPadding
 402         return ['-s', '%sx%s' % (TIVO_WIDTH, endHeight),
 403                 '-padtop', str(topPadding),
 404                 '-padbottom', str(bottomPadding)]
 405
 406 def pad_LR(TIVO_WIDTH, TIVO_HEIGHT, multiplier, vInfo):
 407     endWidth = int((TIVO_HEIGHT * vInfo['vWidth']) /
 408                    (vInfo['vHeight'] * multiplier))
 409     if endWidth % 2:
 410         endWidth -= 1
 411     leftPadding = (TIVO_WIDTH - endWidth) / 2
 412     if leftPadding % 2:
 413         leftPadding -= 1
 414     newpad = pad_check()
 415     if newpad:
 416         return ['-vf', 'scale=%d:%d,pad=%d:%d:%d:0' % (endWidth,
 417                 TIVO_HEIGHT, TIVO_WIDTH, TIVO_HEIGHT, leftPadding)]
 418     else:
 419         rightPadding = (TIVO_WIDTH - endWidth) - leftPadding
 420         return ['-s', '%sx%s' % (endWidth, TIVO_HEIGHT),
 421                 '-padleft', str(leftPadding),
 422                 '-padright', str(rightPadding)]
 423
 424 def select_aspect(inFile, tsn = ''):
 425     TIVO_WIDTH = config.getTivoWidth(tsn)
 426     TIVO_HEIGHT = config.getTivoHeight(tsn)
 427
 428     vInfo = video_info(inFile)
 429
 430     debug('tsn: %s' % tsn)
 431
 432     aspect169 = config.get169Setting(tsn)
 433
 434     debug('aspect169: %s' % aspect169)
 435
 436     optres = config.getOptres(tsn)
 437
 438     debug('optres: %s' % optres)
 439
 440     if optres:
 441         optHeight = config.nearestTivoHeight(vInfo['vHeight'])
 442         optWidth = config.nearestTivoWidth(vInfo['vWidth'])
 443         if optHeight < TIVO_HEIGHT:
 444             TIVO_HEIGHT = optHeight
 445         if optWidth < TIVO_WIDTH:
 446             TIVO_WIDTH = optWidth
 447
 448     if vInfo.get('par2'):
 449         par2 = vInfo['par2']
 450     elif vInfo.get('par'):
 451         par2 = float(vInfo['par'])
 452     else:
 453         # Assume PAR = 1.0
 454         par2 = 1.0
 455
 456     debug(('File=%s vCodec=%s vWidth=%s vHeight=%s vFps=%s millisecs=%s ' +
 457            'TIVO_HEIGHT=%s TIVO_WIDTH=%s') % (inFile, vInfo['vCodec'],
 458           vInfo['vWidth'], vInfo['vHeight'], vInfo['vFps'],
 459           vInfo['millisecs'], TIVO_HEIGHT, TIVO_WIDTH))
 460
 461     if config.isHDtivo(tsn) and not optres:
 462         if vInfo['par']:
 463             npar = par2
 464
 465             # adjust for pixel aspect ratio, if set
 466
 467             if npar < 1.0:
 468                 return ['-s', '%dx%d' % (vInfo['vWidth'],
 469                                          math.ceil(vInfo['vHeight'] / npar))]
 470             elif npar > 1.0:
 471                 # FFMPEG expects width to be a multiple of two
 472                 return ['-s', '%dx%d' % (math.ceil(vInfo['vWidth']*npar/2.0)*2,
 473                                          vInfo['vHeight'])]
 474
 475         if vInfo['vHeight'] <= TIVO_HEIGHT:
 476             # pass all resolutions to S3, except heights greater than
 477             # conf height
 478             return []
 479         # else, resize video.
 480
 481     d = gcd(vInfo['vHeight'], vInfo['vWidth'])
 482     rheight, rwidth = vInfo['vHeight'] / d, vInfo['vWidth'] / d
 483     debug('rheight=%s rwidth=%s' % (rheight, rwidth))
 484
 485     if (rwidth, rheight) in [(1, 1)] and vInfo['par1'] == '8:9':
 486         debug('File + PAR is within 4:3.')
 487         return ['-aspect', '4:3', '-s', '%sx%s' % (TIVO_WIDTH, TIVO_HEIGHT)]
 488
 489     elif ((rwidth, rheight) in [(4, 3), (10, 11), (15, 11), (59, 54),
 490                                 (59, 72), (59, 36), (59, 54)] or
 491           vInfo['dar1'] == '4:3'):
 492         debug('File is within 4:3 list.')
 493         return ['-aspect', '4:3', '-s', '%sx%s' % (TIVO_WIDTH, TIVO_HEIGHT)]
 494
 495     elif (((rwidth, rheight) in [(16, 9), (20, 11), (40, 33), (118, 81),
 496                                 (59, 27)] or vInfo['dar1'] == '16:9')
 497           and (aspect169 or config.get169Letterbox(tsn))):
 498         debug('File is within 16:9 list and 16:9 allowed.')
 499
 500         if config.get169Blacklist(tsn) or (aspect169 and
 501                                            config.get169Letterbox(tsn)):
 502             aspect = '4:3'
 503         else:
 504             aspect = '16:9'
 505         return ['-aspect', aspect, '-s', '%sx%s' % (TIVO_WIDTH, TIVO_HEIGHT)]
 506
 507     else:
 508         settings = ['-aspect']
 509
 510         multiplier16by9 = (16.0 * TIVO_HEIGHT) / (9.0 * TIVO_WIDTH) / par2
 511         multiplier4by3  =  (4.0 * TIVO_HEIGHT) / (3.0 * TIVO_WIDTH) / par2
 512         ratio = vInfo['vWidth'] * 100 * par2 / vInfo['vHeight']
 513         debug('par2=%.3f ratio=%.3f mult4by3=%.3f' % (par2, ratio,
 514                                                       multiplier4by3))
 515
 516         # If video is wider than 4:3 add top and bottom padding
 517
 518         if ratio > 133: # Might be 16:9 file, or just need padding on
 519                         # top and bottom
 520
 521             if aspect169 and ratio > 135: # If file would fall in 4:3
 522                                           # assume it is supposed to be 4:3
 523
 524                 if (config.get169Blacklist(tsn) or
 525                     config.get169Letterbox(tsn)):
 526                     settings.append('4:3')
 527                 else:
 528                     settings.append('16:9')
 529
 530                 if ratio > 177: # too short needs padding top and bottom
 531                     settings += pad_TB(TIVO_WIDTH, TIVO_HEIGHT,
 532                                        multiplier16by9, vInfo)
 533                     debug(('16:9 aspect allowed, file is wider ' +
 534                            'than 16:9 padding top and bottom\n%s') %
 535                           ' '.join(settings))
 536
 537                 else: # too skinny needs padding on left and right.
 538                     settings += pad_LR(TIVO_WIDTH, TIVO_HEIGHT,
 539                                        multiplier16by9, vInfo)
 540                     debug(('16:9 aspect allowed, file is narrower ' +
 541                            'than 16:9 padding left and right\n%s') %
 542                           ' '.join(settings))
 543
 544             else: # this is a 4:3 file or 16:9 output not allowed
 545                 if ratio > 135 and config.get169Letterbox(tsn):
 546                     settings.append('16:9')
 547                     multiplier = multiplier16by9
 548                 else:
 549                     settings.append('4:3')
 550                     multiplier = multiplier4by3
 551                 settings += pad_TB(TIVO_WIDTH, TIVO_HEIGHT,
 552                                    multiplier, vInfo)
 553                 debug(('File is wider than 4:3 padding ' +
 554                        'top and bottom\n%s') % ' '.join(settings))
 555
 556         # If video is taller than 4:3 add left and right padding, this
 557         # is rare. All of these files will always be sent in an aspect
 558         # ratio of 4:3 since they are so narrow.
 559
 560         else:
 561             settings.append('4:3')
 562             settings += pad_LR(TIVO_WIDTH, TIVO_HEIGHT, multiplier4by3, vInfo)
 563             debug('File is taller than 4:3 padding left and right\n%s'
 564                   % ' '.join(settings))
 565
 566         return settings
 567
 568 def tivo_compatible_video(vInfo, tsn, mime=''):
 569     message = (True, '')
 570     while True:
 571         codec = vInfo.get('vCodec', '')
 572         if mime == 'video/mp4':
 573             if codec != 'h264':
 574                 message = (False, 'vCodec %s not compatible' % codec)
 575
 576             break
 577
 578         if mime == 'video/bif':
 579             if codec != 'vc1':
 580                 message = (False, 'vCodec %s not compatible' % codec)
 581
 582             break
 583
 584         if mime == 'video/x-tivo-mpeg-ts':
 585             if codec not in ('h264', 'mpeg2video'):
 586                 message = (False, 'vCodec %s not compatible' % codec)
 587
 588             break
 589
 590         if codec not in ('mpeg2video', 'mpeg1video'):
 591             message = (False, 'vCodec %s not compatible' % codec)
 592             break
 593
 594         if vInfo['kbps'] != None:
 595             abit = max('0', vInfo['aKbps'])
 596             if (int(vInfo['kbps']) - int(abit) >
 597                 config.strtod(config.getMaxVideoBR(tsn)) / 1000):
 598                 message = (False, '%s kbps exceeds max video bitrate' %
 599                                   vInfo['kbps'])
 600                 break
 601         else:
 602             message = (False, '%s kbps not supported' % vInfo['kbps'])
 603             break
 604
 605         if config.isHDtivo(tsn):
 606             # HD Tivo detected, skipping remaining tests.
 607             break
 608
 609         if not vInfo['vFps'] in ['29.97', '59.94']:
 610             message = (False, '%s vFps, should be 29.97' % vInfo['vFps'])
 611             break
 612
 613         if ((config.get169Blacklist(tsn) and not config.get169Setting(tsn))
 614             or (config.get169Letterbox(tsn) and config.get169Setting(tsn))):
 615             if vInfo['dar1'] and vInfo['dar1'] not in ('4:3', '8:9', '880:657'):
 616                 message = (False, ('DAR %s not supported ' +
 617                                    'by BLACKLIST_169 tivos') % vInfo['dar1'])
 618                 break
 619
 620         mode = (vInfo['vWidth'], vInfo['vHeight'])
 621         if mode not in [(720, 480), (704, 480), (544, 480),
 622                         (528, 480), (480, 480), (352, 480), (352, 240)]:
 623             message = (False, '%s x %s not in supported modes' % mode)
 624         break
 625
 626     return message
 627
 628 def tivo_compatible_audio(vInfo, inFile, tsn, mime=''):
 629     message = (True, '')
 630     while True:
 631         codec = vInfo.get('aCodec', '')
 632
 633         if codec == None:
 634             debug('No audio stream detected')
 635             break
 636
 637         if mime == 'video/mp4':
 638             if codec not in ('mpeg4aac', 'libfaad', 'mp4a', 'aac',
 639                              'ac3', 'liba52'):
 640                 message = (False, 'aCodec %s not compatible' % codec)
 641                 break
 642             if vInfo['aCodec'] in ('mpeg4aac', 'libfaad', 'mp4a', 'aac') and (vInfo['aCh'] == None or vInfo['aCh'] > 2):
 643                 message = (False, 'aCodec %s is only supported with 2 or less channels, the track has %s channels' % (codec, vInfo['aCh']))
 644                 break
 645
 646             audio_lang = config.get_tsn('audio_lang', tsn)
 647             if audio_lang:
 648                 if vInfo['mapAudio'][0][0] != select_audiolang(inFile, tsn)[-3:]:
 649                     message = (False, '%s preferred audio track exists' %
 650                                       audio_lang)
 651             break
 652
 653         if mime == 'video/bif':
 654             if codec != 'wmav2':
 655                 message = (False, 'aCodec %s not compatible' % codec)
 656
 657             break
 658
 659         if inFile[-5:].lower() == '.tivo':
 660             break
 661
 662         if mime == 'video/x-tivo-mpeg-ts':
 663             if codec not in ('ac3', 'liba52', 'mp2', 'aac_latm'):
 664                 message = (False, 'aCodec %s not compatible' % codec)
 665
 666             break
 667
 668         if codec not in ('ac3', 'liba52', 'mp2'):
 669             message = (False, 'aCodec %s not compatible' % codec)
 670             break
 671
 672         if (not vInfo['aKbps'] or
 673             int(vInfo['aKbps']) > config.getMaxAudioBR(tsn)):
 674             message = (False, '%s kbps exceeds max audio bitrate' %
 675                               vInfo['aKbps'])
 676             break
 677
 678         audio_lang = config.get_tsn('audio_lang', tsn)
 679         if audio_lang:
 680             if vInfo['mapAudio'][0][0] != select_audiolang(inFile, tsn)[-3:]:
 681                 message = (False, '%s preferred audio track exists' %
 682                                   audio_lang)
 683         break
 684
 685     return message
 686
 687 def tivo_compatible_container(vInfo, inFile, mime=''):
 688     message = (True, '')
 689     container = vInfo.get('container', '')
 690     if ((mime == 'video/mp4' and
 691          (container != 'mov' or inFile.lower().endswith('.mov'))) or
 692         (mime == 'video/bif' and container != 'asf') or
 693         (mime == 'video/x-tivo-mpeg-ts' and container != 'mpegts') or
 694         (mime in ['video/x-tivo-mpeg', 'video/mpeg', ''] and
 695          (container != 'mpeg' or vInfo['vCodec'] == 'mpeg1video'))):
 696         message = (False, 'container %s not compatible' % container)
 697
 698     return message
 699
 700 def mp4_remuxable(inFile, tsn=''):
 701     vInfo = video_info(inFile)
 702     return tivo_compatible_video(vInfo, tsn, 'video/mp4')[0]
 703
 704 def mp4_remux(inFile, basename, tsn=''):
 705     outFile = inFile + '.pyTivo-temp'
 706     newname = basename + '.pyTivo-temp'
 707     if os.path.exists(outFile):
 708         return None  # ugh!
 709
 710     ffmpeg_path = config.get_bin('ffmpeg')
 711     fname = unicode(inFile, 'utf-8')
 712     oname = unicode(outFile, 'utf-8')
 713     if mswindows:
 714         fname = fname.encode('iso8859-1')
 715         oname = oname.encode('iso8859-1')
 716
 717     settings = {'video_codec': '-vcodec copy',
 718             'video_br': select_videobr(inFile, tsn),
 719             'video_fps': select_videofps(inFile, tsn),
 720             'max_video_br': select_maxvideobr(tsn),
 721             'buff_size': select_buffsize(tsn),
 722             'aspect_ratio': ' '.join(select_aspect(inFile, tsn)),
 723             'audio_br': select_audiobr(tsn),
 724             'audio_fr': select_audiofr(inFile, tsn),
 725             'audio_ch': select_audioch(inFile, tsn),
 726             'audio_codec': select_audiocodec(False, inFile, tsn, 'video/mp4'),
 727             'audio_lang': select_audiolang(inFile, tsn),
 728             'ffmpeg_pram': select_ffmpegprams(tsn),
 729             'format': '-f mp4'}
 730
 731     cmd_string = config.getFFmpegTemplate(tsn) % settings
 732     cmd = [ffmpeg_path, '-i', fname] + cmd_string.split() + [oname]
 733
 734     debug('transcoding to tivo model ' + tsn[:3] + ' using ffmpeg command:')
 735     debug(' '.join(cmd))
 736
 737     ffmpeg = subprocess.Popen(cmd)
 738     debug('remuxing ' + inFile + ' to ' + outFile)
 739     if ffmpeg.wait():
 740         debug('error during remuxing')
 741         os.remove(outFile)
 742         return None
 743
 744     return newname
 745
 746 def tivo_compatible(inFile, tsn='', mime=''):
 747     vInfo = video_info(inFile)
 748
 749     message = (True, 'all compatible')
 750     if not config.get_bin('ffmpeg'):
 751         if mime not in ['video/x-tivo-mpeg', 'video/mpeg', '']:
 752             message = (False, 'no ffmpeg')
 753         return message
 754
 755     while True:
 756         vmessage = tivo_compatible_video(vInfo, tsn, mime)
 757         if not vmessage[0]:
 758             message = vmessage
 759             break
 760
 761         amessage = tivo_compatible_audio(vInfo, inFile, tsn, mime)
 762         if not amessage[0]:
 763             message = amessage
 764             break
 765
 766         cmessage = tivo_compatible_container(vInfo, inFile, mime)
 767         if not cmessage[0]:
 768             message = cmessage
 769
 770         break
 771
 772     debug('TRANSCODE=%s, %s, %s' % (['YES', 'NO'][message[0]],
 773                                            message[1], inFile))
 774     return message
 775
 776 def video_info(inFile, cache=True):
 777     vInfo = dict()
 778     fname = unicode(inFile, 'utf-8')
 779     mtime = os.stat(fname).st_mtime
 780     if cache:
 781         if inFile in info_cache and info_cache[inFile][0] == mtime:
 782             debug('CACHE HIT! %s' % inFile)
 783             return info_cache[inFile][1]
 784
 785     vInfo['Supported'] = True
 786
 787     ffmpeg_path = config.get_bin('ffmpeg')
 788     if not ffmpeg_path:
 789         if os.path.splitext(inFile)[1].lower() not in ['.mpg', '.mpeg',
 790                                                        '.vob', '.tivo']:
 791             vInfo['Supported'] = False
 792         vInfo.update({'millisecs': 0, 'vWidth': 704, 'vHeight': 480,
 793                       'rawmeta': {}})
 794         if cache:
 795             info_cache[inFile] = (mtime, vInfo)
 796         return vInfo
 797
 798     if mswindows:
 799         fname = fname.encode('iso8859-1')
 800     cmd = [ffmpeg_path, '-i', fname]
 801     # Windows and other OS buffer 4096 and ffmpeg can output more than that.
 802     err_tmp = tempfile.TemporaryFile()
 803     ffmpeg = subprocess.Popen(cmd, stderr=err_tmp, stdout=subprocess.PIPE,
 804                               stdin=subprocess.PIPE)
 805
 806     # wait configured # of seconds: if ffmpeg is not back give up
 807     limit = config.getFFmpegWait()
 808     if limit:
 809         for i in xrange(limit * 20):
 810             time.sleep(.05)
 811             if not ffmpeg.poll() == None:
 812                 break
 813
 814         if ffmpeg.poll() == None:
 815             kill(ffmpeg)
 816             vInfo['Supported'] = False
 817             if cache:
 818                 info_cache[inFile] = (mtime, vInfo)
 819             return vInfo
 820     else:
 821         ffmpeg.wait()
 822
 823     err_tmp.seek(0)
 824     output = err_tmp.read()
 825     err_tmp.close()
 826     debug('ffmpeg output=%s' % output)
 827
 828     attrs = {'container': r'Input #0, ([^,]+),',
 829              'vCodec': r'Video: ([^, ]+)',             # video codec
 830              'aKbps': r'.*Audio: .+, (.+) (?:kb/s).*',     # audio bitrate
 831              'aCodec': r'.*Audio: ([^, ]+)',             # audio codec
 832              'aFreq': r'.*Audio: .+, (.+) (?:Hz).*',       # audio frequency
 833              'mapVideo': r'([0-9]+[.:]+[0-9]+).*: Video:.*'}  # video mapping
 834
 835     for attr in attrs:
 836         rezre = re.compile(attrs[attr])
 837         x = rezre.search(output)
 838         if x:
 839             vInfo[attr] = x.group(1)
 840         else:
 841             if attr in ['container', 'vCodec']:
 842                 vInfo[attr] = ''
 843                 vInfo['Supported'] = False
 844             else:
 845                 vInfo[attr] = None
 846             debug('failed at ' + attr)
 847
 848     rezre = re.compile(r'.*Audio: .+, (?:(\d+)(?:(?:\.(\d).*)?(?: channels.*)?)|(stereo|mono)),.*')
 849     x = rezre.search(output)
 850     if x:
 851         if x.group(3):
 852             if x.group(3) == 'stereo':
 853                 vInfo['aCh'] = 2
 854             elif x.group(3) == 'mono':
 855                 vInfo['aCh'] = 1
 856         elif x.group(2):
 857             vInfo['aCh'] = int(x.group(1)) + int(x.group(2))
 858         elif x.group(1):
 859             vInfo['aCh'] = int(x.group(1))
 860         else:
 861             vInfo['aCh'] = None
 862             debug('failed at aCh')
 863     else:
 864         vInfo['aCh'] = None
 865         debug('failed at aCh')
 866
 867     rezre = re.compile(r'.*Video: .+, (\d+)x(\d+)[, ].*')
 868     x = rezre.search(output)
 869     if x:
 870         vInfo['vWidth'] = int(x.group(1))
 871         vInfo['vHeight'] = int(x.group(2))
 872     else:
 873         vInfo['vWidth'] = ''
 874         vInfo['vHeight'] = ''
 875         vInfo['Supported'] = False
 876         debug('failed at vWidth/vHeight')
 877
 878     rezre = re.compile(r'.*Video: .+, (.+) (?:fps|tb\(r\)|tbr).*')
 879     x = rezre.search(output)
 880     if x:
 881         vInfo['vFps'] = x.group(1)
 882         if '.' not in vInfo['vFps']:
 883             vInfo['vFps'] += '.00'
 884
 885         # Allow override only if it is mpeg2 and frame rate was doubled
 886         # to 59.94
 887
 888         if vInfo['vCodec'] == 'mpeg2video' and vInfo['vFps'] != '29.97':
 889             # First look for the build 7215 version
 890             rezre = re.compile(r'.*film source: 29.97.*')
 891             x = rezre.search(output.lower())
 892             if x:
 893                 debug('film source: 29.97 setting vFps to 29.97')
 894                 vInfo['vFps'] = '29.97'
 895             else:
 896                 # for build 8047:
 897                 rezre = re.compile(r'.*frame rate differs from container ' +
 898                                    r'frame rate: 29.97.*')
 899                 debug('Bug in VideoReDo')
 900                 x = rezre.search(output.lower())
 901                 if x:
 902                     vInfo['vFps'] = '29.97'
 903     else:
 904         vInfo['vFps'] = ''
 905         vInfo['Supported'] = False
 906         debug('failed at vFps')
 907
 908     durre = re.compile(r'.*Duration: ([0-9]+):([0-9]+):([0-9]+)\.([0-9]+),')
 909     d = durre.search(output)
 910
 911     if d:
 912         vInfo['millisecs'] = ((int(d.group(1)) * 3600 +
 913                                int(d.group(2)) * 60 +
 914                                int(d.group(3))) * 1000 +
 915                               int(d.group(4)) * (10 ** (3 - len(d.group(4)))))
 916     else:
 917         vInfo['millisecs'] = 0
 918
 919     # get bitrate of source for tivo compatibility test.
 920     rezre = re.compile(r'.*bitrate: (.+) (?:kb/s).*')
 921     x = rezre.search(output)
 922     if x:
 923         vInfo['kbps'] = x.group(1)
 924     else:
 925         # Fallback method of getting video bitrate
 926         # Sample line:  Stream #0.0[0x1e0]: Video: mpeg2video, yuv420p,
 927         #               720x480 [PAR 32:27 DAR 16:9], 9800 kb/s, 59.94 tb(r)
 928         rezre = re.compile(r'.*Stream #0\.0\[.*\]: Video: mpeg2video, ' +
 929                            r'\S+, \S+ \[.*\], (\d+) (?:kb/s).*')
 930         x = rezre.search(output)
 931         if x:
 932             vInfo['kbps'] = x.group(1)
 933         else:
 934             vInfo['kbps'] = None
 935             debug('failed at kbps')
 936
 937     # get par.
 938     rezre = re.compile(r'.*Video: .+PAR ([0-9]+):([0-9]+) DAR [0-9:]+.*')
 939     x = rezre.search(output)
 940     if x and x.group(1) != "0" and x.group(2) != "0":
 941         vInfo['par1'] = x.group(1) + ':' + x.group(2)
 942         vInfo['par2'] = float(x.group(1)) / float(x.group(2))
 943     else:
 944         vInfo['par1'], vInfo['par2'] = None, None
 945
 946     # get dar.
 947     rezre = re.compile(r'.*Video: .+DAR ([0-9]+):([0-9]+).*')
 948     x = rezre.search(output)
 949     if x and x.group(1) != "0" and x.group(2) != "0":
 950         vInfo['dar1'] = x.group(1) + ':' + x.group(2)
 951     else:
 952         vInfo['dar1'] = None
 953
 954     # get Audio Stream mapping.
 955     rezre = re.compile(r'([0-9]+[.:]+[0-9]+)(.*): Audio:(.*)')
 956     x = rezre.search(output)
 957     amap = []
 958     if x:
 959         for x in rezre.finditer(output):
 960             amap.append((x.group(1), x.group(2) + x.group(3)))
 961     else:
 962         amap.append(('', ''))
 963         debug('failed at mapAudio')
 964     vInfo['mapAudio'] = amap
 965
 966     vInfo['par'] = None
 967
 968     # get Metadata dump (newer ffmpeg).
 969     lines = output.split('\n')
 970     rawmeta = {}
 971     flag = False
 972
 973     for line in lines:
 974         if line.startswith('  Metadata:'):
 975             flag = True
 976         else:
 977             if flag:
 978                 if line.startswith('  Duration:'):
 979                     flag = False
 980                 else:
 981                     try:
 982                         key, value = [x.strip() for x in line.split(':', 1)]
 983                         try:
 984                             value = value.decode('utf-8')
 985                         except:
 986                             if sys.platform == 'darwin':
 987                                 value = value.decode('macroman')
 988                             else:
 989                                 value = value.decode('iso8859-1')
 990                         rawmeta[key] = [value]
 991                     except:
 992                         pass
 993
 994     vInfo['rawmeta'] = rawmeta
 995
 996     data = metadata.from_text(inFile)
 997     for key in data:
 998         if key.startswith('Override_'):
 999             vInfo['Supported'] = True
1000             if key.startswith('Override_mapAudio'):
1001                 audiomap = dict(vInfo['mapAudio'])
1002                 stream = key.replace('Override_mapAudio', '').strip()
1003                 if stream in audiomap:
1004                     newaudiomap = (stream, data[key])
1005                     audiomap.update([newaudiomap])
1006                     vInfo['mapAudio'] = sorted(audiomap.items(),
1007                                                key=lambda (k,v): (k,v))
1008             elif key.startswith('Override_millisecs'):
1009                 vInfo[key.replace('Override_', '')] = int(data[key])
1010             else:
1011                 vInfo[key.replace('Override_', '')] = data[key]
1012
1013     if cache:
1014         info_cache[inFile] = (mtime, vInfo)
1015     debug("; ".join(["%s=%s" % (k, v) for k, v in vInfo.items()]))
1016     return vInfo
1017
1018 def audio_check(inFile, tsn):
1019     cmd_string = ('-y -vcodec mpeg2video -r 29.97 -b 1000k -acodec copy ' +
1020                   select_audiolang(inFile, tsn) + ' -t 00:00:01 -f vob -')
1021     fname = unicode(inFile, 'utf-8')
1022     if mswindows:
1023         fname = fname.encode('iso8859-1')
1024     cmd = [config.get_bin('ffmpeg'), '-i', fname] + cmd_string.split()
1025     ffmpeg = subprocess.Popen(cmd, stdout=subprocess.PIPE)
1026     fd, testname = tempfile.mkstemp()
1027     testfile = os.fdopen(fd, 'wb')
1028     try:
1029         shutil.copyfileobj(ffmpeg.stdout, testfile)
1030     except:
1031         kill(ffmpeg)
1032         testfile.close()
1033         vInfo = None
1034     else:
1035         testfile.close()
1036         vInfo = video_info(testname, False)
1037     os.remove(testname)
1038     return vInfo
1039
1040 def supported_format(inFile):
1041     if video_info(inFile)['Supported']:
1042         return True
1043     else:
1044         debug('FALSE, file not supported %s' % inFile)
1045         return False
1046
1047 def kill(popen):
1048     debug('killing pid=%s' % str(popen.pid))
1049     if mswindows:
1050         win32kill(popen.pid)
1051     else:
1052         import os, signal
1053         for i in xrange(3):
1054             debug('sending SIGTERM to pid: %s' % popen.pid)
1055             os.kill(popen.pid, signal.SIGTERM)
1056             time.sleep(.5)
1057             if popen.poll() is not None:
1058                 debug('process %s has exited' % popen.pid)
1059                 break
1060         else:
1061             while popen.poll() is None:
1062                 debug('sending SIGKILL to pid: %s' % popen.pid)
1063                 os.kill(popen.pid, signal.SIGKILL)
1064                 time.sleep(.5)
1065
1066 def win32kill(pid):
1067     import ctypes
1068     handle = ctypes.windll.kernel32.OpenProcess(1, False, pid)
1069     ctypes.windll.kernel32.TerminateProcess(handle, -1)
1070     ctypes.windll.kernel32.CloseHandle(handle)
1071
1072 def gcd(a, b):
1073     while b:
1074         a, b = b, a % b
1075     return a