plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 logger = logging.getLogger('pyTivo.video.video')
  25
  26 SCRIPTDIR = os.path.dirname(__file__)
  27
  28 CLASS_NAME = 'Video'
  29
  30 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  31
  32 # Preload the templates
  33 def tmpl(name):
  34     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  35
  36 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  37 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  38 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  39
  40 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  41 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  42 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  43 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  44 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  45 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  46 .wm .wmd .wtv .yuv""".split()
  47
  48 use_extensions = True
  49 try:
  50     assert(config.get_bin('ffmpeg'))
  51 except:
  52     use_extensions = False
  53
  54 queue = []  # Recordings to push
  55
  56 def uniso(iso):
  57     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  58
  59 def isodt(iso):
  60     return datetime(*uniso(iso)[:6])
  61
  62 def isogm(iso):
  63     return int(calendar.timegm(uniso(iso)))
  64
  65 class Pushable(object):
  66
  67     def push_one_file(self, f):
  68         file_info = VideoDetails()
  69         file_info['valid'] = transcode.supported_format(f['path'])
  70
  71         mime = 'video/mpeg'
  72         if config.isHDtivo(f['tsn']):
  73             for m in ['video/mp4', 'video/bif']:
  74                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  75                     mime = m
  76                     break
  77
  78             if (mime == 'video/mpeg' and
  79                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  80                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'])
  81                 if new_path:
  82                     mime = 'video/mp4'
  83                     f['name'] = new_path
  84
  85         if file_info['valid']:
  86             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
  87
  88         url = f['url'] + quote(f['name'])
  89
  90         title = file_info['seriesTitle']
  91         if not title:
  92             title = file_info['title']
  93
  94         source = file_info['seriesId']
  95         if not source:
  96             source = title
  97
  98         subtitle = file_info['episodeTitle']
  99         try:
 100             m = mind.getMind(f['tsn'])
 101             m.pushVideo(
 102                 tsn = f['tsn'],
 103                 url = url,
 104                 description = file_info['description'],
 105                 duration = file_info['duration'] / 1000,
 106                 size = file_info['size'],
 107                 title = title,
 108                 subtitle = subtitle,
 109                 source = source,
 110                 mime = mime,
 111                 tvrating = file_info['tvRating'])
 112         except Exception, msg:
 113             logger.error(msg)
 114
 115     def process_queue(self):
 116         while queue:
 117             time.sleep(5)
 118             item = queue.pop(0)
 119             self.push_one_file(item)
 120
 121     def readip(self):
 122         """ returns your external IP address by querying dyndns.org """
 123         f = urllib.urlopen('http://checkip.dyndns.org/')
 124         s = f.read()
 125         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 126         return m.group(0)
 127
 128     def Push(self, handler, query):
 129         try:
 130             tsn = query['tsn'][0]
 131         except:
 132             logger.error('Push requires a TiVo Service Number')
 133             handler.send_error(404)
 134             return
 135
 136         if not tsn in config.tivos:
 137             for key, value in config.tivos.items():
 138                 if value.get('name') == tsn:
 139                     tsn = key
 140                     break
 141         try:
 142             tivo_name = config.tivos[tsn]['name']
 143         except:
 144             tivo_name = tsn
 145
 146         container = quote(query['Container'][0].split('/')[0])
 147         ip = config.get_ip(tsn)
 148         port = config.getPort()
 149
 150         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 151         if config.getIsExternal(tsn):
 152             exturl = config.get_server('externalurl')
 153             if exturl:
 154                 if not exturl.endswith('/'):
 155                     exturl += '/'
 156                 baseurl = exturl + container
 157             else:
 158                 ip = self.readip()
 159                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 160
 161         path = self.get_local_base_path(handler, query)
 162
 163         files = query.get('File', [])
 164         for f in files:
 165             file_path = os.path.normpath(path + '/' + f)
 166             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 167                           'url': baseurl})
 168             if len(queue) == 1:
 169                 thread.start_new_thread(Video.process_queue, (self,))
 170
 171             logger.info('[%s] Queued "%s" for Push to %s' %
 172                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 173                          unicode(file_path, 'utf-8'), tivo_name))
 174
 175         files = [unicode(f, 'utf-8') for f in files]
 176         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 177
 178 class BaseVideo(Plugin):
 179
 180     CONTENT_TYPE = 'x-container/tivo-videos'
 181
 182     tvbus_cache = LRUCache(1)
 183
 184     def video_file_filter(self, full_path, type=None):
 185         if os.path.isdir(unicode(full_path, 'utf-8')):
 186             return True
 187         if use_extensions:
 188             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 189         else:
 190             return transcode.supported_format(full_path)
 191
 192     def send_file(self, handler, path, query):
 193         mime = 'video/x-tivo-mpeg'
 194         tsn = handler.headers.getheader('tsn', '')
 195         try:
 196             assert(tsn)
 197             tivo_name = config.tivos[tsn].get('name', tsn)
 198         except:
 199             tivo_name = handler.address_string()
 200
 201         is_tivo_file = (path[-5:].lower() == '.tivo')
 202
 203         if 'Format' in query:
 204             mime = query['Format'][0]
 205
 206         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 207         compatible = (not needs_tivodecode and
 208                       transcode.tivo_compatible(path, tsn, mime)[0])
 209
 210         try:  # "bytes=XXX-"
 211             offset = int(handler.headers.getheader('Range')[6:-1])
 212         except:
 213             offset = 0
 214
 215         if needs_tivodecode:
 216             valid = bool(config.get_bin('tivodecode') and
 217                          config.get_server('tivo_mak'))
 218         else:
 219             valid = True
 220
 221         if valid and offset:
 222             valid = ((compatible and offset < os.path.getsize(path)) or
 223                      (not compatible and transcode.is_resumable(path, offset)))
 224
 225         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 226         faking = (mime == 'video/x-tivo-mpeg' and
 227                   not (is_tivo_file and compatible))
 228         fname = unicode(path, 'utf-8')
 229         thead = ''
 230         if faking:
 231             thead = self.tivo_header(tsn, path, mime)
 232         if compatible:
 233             size = os.path.getsize(fname) + len(thead)
 234             handler.send_response(200)
 235             handler.send_header('Content-Length', size - offset)
 236             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 237                                 (offset, size - offset - 1, size))
 238         else:
 239             handler.send_response(206)
 240             handler.send_header('Transfer-Encoding', 'chunked')
 241         handler.send_header('Content-Type', mime)
 242         handler.end_headers()
 243
 244         logger.info('[%s] Start sending "%s" to %s' %
 245                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 246         start = time.time()
 247         count = 0
 248
 249         if valid:
 250             if compatible:
 251                 if faking and not offset:
 252                     handler.wfile.write(thead)
 253                 logger.debug('"%s" is tivo compatible' % fname)
 254                 f = open(fname, 'rb')
 255                 try:
 256                     if mime == 'video/mp4':
 257                         count = qtfaststart.process(f, handler.wfile, offset)
 258                     else:
 259                         if offset:
 260                             offset -= len(thead)
 261                             f.seek(offset)
 262                         while True:
 263                             block = f.read(512 * 1024)
 264                             if not block:
 265                                 break
 266                             handler.wfile.write(block)
 267                             count += len(block)
 268                 except Exception, msg:
 269                     logger.info(msg)
 270                 f.close()
 271             else:
 272                 logger.debug('"%s" is not tivo compatible' % fname)
 273                 if offset:
 274                     count = transcode.resume_transfer(path, handler.wfile,
 275                                                       offset)
 276                 else:
 277                     count = transcode.transcode(False, path, handler.wfile,
 278                                                 tsn, mime, thead)
 279         try:
 280             if not compatible:
 281                  handler.wfile.write('0\r\n\r\n')
 282             handler.wfile.flush()
 283         except Exception, msg:
 284             logger.info(msg)
 285
 286         mega_elapsed = (time.time() - start) * 1024 * 1024
 287         if mega_elapsed < 1:
 288             mega_elapsed = 1
 289         rate = count * 8.0 / mega_elapsed
 290         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 291                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 292                      tivo_name, count, rate))
 293
 294         if fname.endswith('.pyTivo-temp'):
 295             os.remove(fname)
 296
 297     def __duration(self, full_path):
 298         return transcode.video_info(full_path)['millisecs']
 299
 300     def __total_items(self, full_path):
 301         count = 0
 302         try:
 303             full_path = unicode(full_path, 'utf-8')
 304             for f in os.listdir(full_path):
 305                 if f.startswith('.'):
 306                     continue
 307                 f = os.path.join(full_path, f)
 308                 f2 = f.encode('utf-8')
 309                 if os.path.isdir(f):
 310                     count += 1
 311                 elif use_extensions:
 312                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 313                         count += 1
 314                 elif f2 in transcode.info_cache:
 315                     if transcode.supported_format(f2):
 316                         count += 1
 317         except:
 318             pass
 319         return count
 320
 321     def __est_size(self, full_path, tsn='', mime=''):
 322         # Size is estimated by taking audio and video bit rate adding 2%
 323
 324         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 325             return os.path.getsize(unicode(full_path, 'utf-8'))
 326         else:
 327             # Must be re-encoded
 328             audioBPS = config.getMaxAudioBR(tsn) * 1000
 329             #audioBPS = config.strtod(config.getAudioBR(tsn))
 330             videoBPS = transcode.select_videostr(full_path, tsn)
 331             bitrate =  audioBPS + videoBPS
 332             return int((self.__duration(full_path) / 1000) *
 333                        (bitrate * 1.02 / 8))
 334
 335     def metadata_full(self, full_path, tsn='', mime='', mtime=None):
 336         data = {}
 337         vInfo = transcode.video_info(full_path)
 338
 339         if ((int(vInfo['vHeight']) >= 720 and
 340              config.getTivoHeight >= 720) or
 341             (int(vInfo['vWidth']) >= 1280 and
 342              config.getTivoWidth >= 1280)):
 343             data['showingBits'] = '4096'
 344
 345         data.update(metadata.basic(full_path, mtime))
 346         if full_path[-5:].lower() == '.tivo':
 347             data.update(metadata.from_tivo(full_path))
 348         if full_path[-4:].lower() == '.wtv':
 349             data.update(metadata.from_mscore(vInfo['rawmeta']))
 350
 351         if 'episodeNumber' in data:
 352             try:
 353                 ep = int(data['episodeNumber'])
 354             except:
 355                 ep = 0
 356             data['episodeNumber'] = str(ep)
 357
 358         if config.getDebug() and 'vHost' not in data:
 359             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 360             if compatible:
 361                 transcode_options = []
 362             else:
 363                 transcode_options = transcode.transcode(True, full_path,
 364                                                         '', tsn, mime)
 365             data['vHost'] = (
 366                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 367                 ['SOURCE INFO: '] +
 368                 ["%s=%s" % (k, v)
 369                  for k, v in sorted(vInfo.items(), reverse=True)] +
 370                 ['TRANSCODE OPTIONS: '] +
 371                 transcode_options +
 372                 ['SOURCE FILE: ', os.path.basename(full_path)]
 373             )
 374
 375         now = datetime.utcnow()
 376         if 'time' in data:
 377             if data['time'].lower() == 'file':
 378                 if not mtime:
 379                     mtime = os.path.getmtime(unicode(full_path, 'utf-8'))
 380                 try:
 381                     now = datetime.utcfromtimestamp(mtime)
 382                 except:
 383                     logger.warning('Bad file time on ' + full_path)
 384             elif data['time'].lower() == 'oad':
 385                     now = isodt(data['originalAirDate'])
 386             else:
 387                 try:
 388                     now = isodt(data['time'])
 389                 except:
 390                     logger.warning('Bad time format: ' + data['time'] +
 391                                    ' , using current time')
 392
 393         duration = self.__duration(full_path)
 394         duration_delta = timedelta(milliseconds = duration)
 395         min = duration_delta.seconds / 60
 396         sec = duration_delta.seconds % 60
 397         hours = min / 60
 398         min = min % 60
 399
 400         data.update({'time': now.isoformat(),
 401                      'startTime': now.isoformat(),
 402                      'stopTime': (now + duration_delta).isoformat(),
 403                      'size': self.__est_size(full_path, tsn, mime),
 404                      'duration': duration,
 405                      'iso_duration': ('P%sDT%sH%sM%sS' %
 406                           (duration_delta.days, hours, min, sec))})
 407
 408         return data
 409
 410     def QueryContainer(self, handler, query):
 411         tsn = handler.headers.getheader('tsn', '')
 412         subcname = query['Container'][0]
 413
 414         if not self.get_local_path(handler, query):
 415             handler.send_error(404)
 416             return
 417
 418         container = handler.container
 419         force_alpha = container.getboolean('force_alpha')
 420         ar = container.get('allow_recurse', 'auto').lower()
 421         if ar == 'auto':
 422             allow_recurse = not tsn or tsn[0] < '7'
 423         else:
 424             allow_recurse = ar in ('1', 'yes', 'true', 'on')
 425         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 426
 427         files, total, start = self.get_files(handler, query,
 428                                              self.video_file_filter,
 429                                              force_alpha, allow_recurse)
 430
 431         videos = []
 432         local_base_path = self.get_local_base_path(handler, query)
 433         for f in files:
 434             video = VideoDetails()
 435             mtime = f.mdate
 436             try:
 437                 ltime = time.localtime(mtime)
 438             except:
 439                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 440                 mtime = time.time()
 441                 ltime = time.localtime(mtime)
 442             video['captureDate'] = hex(int(mtime))
 443             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 444             video['name'] = os.path.basename(f.name)
 445             video['path'] = f.name
 446             video['part_path'] = f.name.replace(local_base_path, '', 1)
 447             if not video['part_path'].startswith(os.path.sep):
 448                 video['part_path'] = os.path.sep + video['part_path']
 449             video['title'] = os.path.basename(f.name)
 450             video['is_dir'] = f.isdir
 451             if video['is_dir']:
 452                 video['small_path'] = subcname + '/' + video['name']
 453                 video['total_items'] = self.__total_items(f.name)
 454             else:
 455                 if len(files) == 1 or f.name in transcode.info_cache:
 456                     video['valid'] = transcode.supported_format(f.name)
 457                     if video['valid']:
 458                         video.update(self.metadata_full(f.name, tsn,
 459                                      mtime=mtime))
 460                         if len(files) == 1:
 461                             video['captureDate'] = hex(isogm(video['time']))
 462                 else:
 463                     video['valid'] = True
 464                     video.update(metadata.basic(f.name, mtime))
 465
 466                 if self.use_ts(tsn, f.name):
 467                     video['mime'] = 'video/x-tivo-mpeg-ts'
 468                 else:
 469                     video['mime'] = 'video/x-tivo-mpeg'
 470
 471                 video['textSize'] = metadata.human_size(f.size)
 472
 473             videos.append(video)
 474
 475         if use_html:
 476             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 477         else:
 478             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 479         t.container = handler.cname
 480         t.name = subcname
 481         t.total = total
 482         t.start = start
 483         t.videos = videos
 484         t.quote = quote
 485         t.escape = escape
 486         t.crc = zlib.crc32
 487         t.guid = config.getGUID()
 488         t.tivos = config.tivos
 489         if use_html:
 490             handler.send_html(str(t))
 491         else:
 492             handler.send_xml(str(t))
 493
 494     def use_ts(self, tsn, file_path):
 495         if config.is_ts_capable(tsn):
 496             if file_path[-5:].lower() == '.tivo':
 497                 try:
 498                     flag = file(file_path).read(8)
 499                 except:
 500                     return False
 501                 if ord(flag[7]) & 0x20:
 502                     return True
 503             elif config.has_ts_flag():
 504                 return True
 505
 506         return False
 507
 508     def get_details_xml(self, tsn, file_path):
 509         if (tsn, file_path) in self.tvbus_cache:
 510             details = self.tvbus_cache[(tsn, file_path)]
 511         else:
 512             file_info = VideoDetails()
 513             file_info['valid'] = transcode.supported_format(file_path)
 514             if file_info['valid']:
 515                 file_info.update(self.metadata_full(file_path, tsn))
 516
 517             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 518             t.video = file_info
 519             t.escape = escape
 520             t.get_tv = metadata.get_tv
 521             t.get_mpaa = metadata.get_mpaa
 522             t.get_stars = metadata.get_stars
 523             t.get_color = metadata.get_color
 524             details = str(t)
 525             self.tvbus_cache[(tsn, file_path)] = details
 526         return details
 527
 528     def tivo_header(self, tsn, path, mime):
 529         def pad(length, align):
 530             extra = length % align
 531             if extra:
 532                 extra = align - extra
 533             return extra
 534
 535         if mime == 'video/x-tivo-mpeg-ts':
 536             flag = 45
 537         else:
 538             flag = 13
 539         details = self.get_details_xml(tsn, path)
 540         ld = len(details)
 541         chunk = details + '\0' * (pad(ld, 4) + 4)
 542         lc = len(chunk)
 543         blocklen = lc * 2 + 40
 544         padding = pad(blocklen, 1024)
 545
 546         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 547                                             padding + blocklen, 2),
 548                         struct.pack('>LLHH', lc + 12, ld, 1, 0),
 549                         chunk,
 550                         struct.pack('>LLHH', lc + 12, ld, 2, 0),
 551                         chunk, '\0' * padding])
 552
 553     def TVBusQuery(self, handler, query):
 554         tsn = handler.headers.getheader('tsn', '')
 555         f = query['File'][0]
 556         path = self.get_local_path(handler, query)
 557         file_path = os.path.normpath(path + '/' + f)
 558
 559         details = self.get_details_xml(tsn, file_path)
 560
 561         handler.send_xml(details)
 562
 563 class Video(BaseVideo, Pushable):
 564         pass
 565
 566 class VideoDetails(DictMixin):
 567
 568     def __init__(self, d=None):
 569         if d:
 570             self.d = d
 571         else:
 572             self.d = {}
 573
 574     def __getitem__(self, key):
 575         if key not in self.d:
 576             self.d[key] = self.default(key)
 577         return self.d[key]
 578
 579     def __contains__(self, key):
 580         return True
 581
 582     def __setitem__(self, key, value):
 583         self.d[key] = value
 584
 585     def __delitem__(self):
 586         del self.d[key]
 587
 588     def keys(self):
 589         return self.d.keys()
 590
 591     def __iter__(self):
 592         return self.d.__iter__()
 593
 594     def iteritems(self):
 595         return self.d.iteritems()
 596
 597     def default(self, key):
 598         defaults = {
 599             'showingBits' : '0',
 600             'displayMajorNumber' : '0',
 601             'displayMinorNumber' : '0',
 602             'isEpisode' : 'true',
 603             'colorCode' : '4',
 604             'showType' : ('SERIES', '5')
 605         }
 606         if key in defaults:
 607             return defaults[key]
 608         elif key.startswith('v'):
 609             return []
 610         else:
 611             return ''