plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 logger = logging.getLogger('pyTivo.video.video')
  25
  26 SCRIPTDIR = os.path.dirname(__file__)
  27
  28 CLASS_NAME = 'Video'
  29
  30 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  31
  32 # Preload the templates
  33 def tmpl(name):
  34     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  35
  36 HTML_CONTAINER_TEMPLATE_MOBILE = tmpl('container_mob.tmpl')
  37 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  38 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  39 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  40
  41 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  42 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  43 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  44 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  45 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  46 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  47 .wm .wmd .wtv .yuv""".split()
  48
  49 use_extensions = True
  50 try:
  51     assert(config.get_bin('ffmpeg'))
  52 except:
  53     use_extensions = False
  54
  55 queue = []  # Recordings to push
  56
  57 def uniso(iso):
  58     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  59
  60 def isodt(iso):
  61     return datetime(*uniso(iso)[:6])
  62
  63 def isogm(iso):
  64     return int(calendar.timegm(uniso(iso)))
  65
  66 class Pushable(object):
  67
  68     def push_one_file(self, f):
  69         file_info = VideoDetails()
  70         file_info['valid'] = transcode.supported_format(f['path'])
  71
  72         temp_share = config.get_server('temp_share', '')
  73         temp_share_path = ''
  74         if temp_share:
  75             for name, data in config.getShares():
  76                 if temp_share == name:
  77                     temp_share_path = data.get('path')
  78
  79         mime = 'video/mpeg'
  80         if config.isHDtivo(f['tsn']):
  81             for m in ['video/mp4', 'video/bif']:
  82                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  83                     mime = m
  84                     break
  85
  86             if (mime == 'video/mpeg' and
  87                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  88                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'], temp_share_path)
  89                 if new_path:
  90                     mime = 'video/mp4'
  91                     f['name'] = new_path
  92                     if temp_share_path:
  93                         ip = config.get_ip()
  94                         port = config.getPort()
  95                         container = quote(temp_share) + '/'
  96                         f['url'] = 'http://%s:%s/%s' % (ip, port, container)
  97
  98         if file_info['valid']:
  99             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
 100
 101         url = f['url'] + quote(f['name'])
 102
 103         title = file_info['seriesTitle']
 104         if not title:
 105             title = file_info['title']
 106
 107         source = file_info['seriesId']
 108         if not source:
 109             source = title
 110
 111         subtitle = file_info['episodeTitle']
 112         try:
 113             m = mind.getMind(f['tsn'])
 114             m.pushVideo(
 115                 tsn = f['tsn'],
 116                 url = url,
 117                 description = file_info['description'],
 118                 duration = file_info['duration'] / 1000,
 119                 size = file_info['size'],
 120                 title = title,
 121                 subtitle = subtitle,
 122                 source = source,
 123                 mime = mime,
 124                 tvrating = file_info['tvRating'])
 125         except Exception, msg:
 126             logger.error(msg)
 127
 128     def process_queue(self):
 129         while queue:
 130             time.sleep(5)
 131             item = queue.pop(0)
 132             self.push_one_file(item)
 133
 134     def readip(self):
 135         """ returns your external IP address by querying dyndns.org """
 136         f = urllib.urlopen('http://checkip.dyndns.org/')
 137         s = f.read()
 138         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 139         return m.group(0)
 140
 141     def Push(self, handler, query):
 142         tsn = query['tsn'][0]
 143         for key in config.tivos:
 144             if config.tivos[key]['name'] == tsn:
 145                 tsn = key
 146                 break
 147         tivo_name = config.tivos[tsn].get('name', tsn)
 148
 149         container = quote(query['Container'][0].split('/')[0])
 150         ip = config.get_ip(tsn)
 151         port = config.getPort()
 152
 153         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 154         if config.getIsExternal(tsn):
 155             exturl = config.get_server('externalurl')
 156             if exturl:
 157                 if not exturl.endswith('/'):
 158                     exturl += '/'
 159                 baseurl = exturl + container
 160             else:
 161                 ip = self.readip()
 162                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 163
 164         path = self.get_local_base_path(handler, query)
 165
 166         files = query.get('File', [])
 167         for f in files:
 168             file_path = os.path.normpath(path + '/' + f)
 169             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 170                           'url': baseurl})
 171             if len(queue) == 1:
 172                 thread.start_new_thread(Video.process_queue, (self,))
 173
 174             logger.info('[%s] Queued "%s" for Push to %s' %
 175                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 176                          unicode(file_path, 'utf-8'), tivo_name))
 177
 178         files = [unicode(f, 'utf-8') for f in files]
 179         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 180
 181 class BaseVideo(Plugin):
 182
 183     CONTENT_TYPE = 'x-container/tivo-videos'
 184
 185     tvbus_cache = LRUCache(1)
 186
 187     def video_file_filter(self, full_path, type=None):
 188         if os.path.isdir(unicode(full_path, 'utf-8')):
 189             return True
 190         if use_extensions:
 191             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 192         else:
 193             return transcode.supported_format(full_path)
 194
 195     def send_file(self, handler, path, query):
 196         mime = 'video/x-tivo-mpeg'
 197         tsn = handler.headers.getheader('tsn', '')
 198         tivo_name = config.tivos[tsn].get('name', tsn)
 199
 200         is_tivo_file = (path[-5:].lower() == '.tivo')
 201
 202         if 'Format' in query:
 203             mime = query['Format'][0]
 204
 205         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 206         compatible = (not needs_tivodecode and
 207                       transcode.tivo_compatible(path, tsn, mime)[0])
 208
 209         try:  # "bytes=XXX-"
 210             offset = int(handler.headers.getheader('Range')[6:-1])
 211         except:
 212             offset = 0
 213
 214         if needs_tivodecode:
 215             valid = bool(config.get_bin('tivodecode') and
 216                          config.get_server('tivo_mak'))
 217         else:
 218             valid = True
 219
 220         if valid and offset:
 221             valid = ((compatible and offset < os.path.getsize(path)) or
 222                      (not compatible and transcode.is_resumable(path, offset)))
 223
 224         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 225         faking = (mime == 'video/x-tivo-mpeg' and
 226                   not (is_tivo_file and compatible))
 227         fname = unicode(path, 'utf-8')
 228         thead = ''
 229         if faking:
 230             thead = self.tivo_header(tsn, path, mime)
 231         if compatible:
 232             size = os.path.getsize(fname) + len(thead)
 233             handler.send_response(200)
 234             handler.send_header('Content-Length', size - offset)
 235             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 236                                 (offset, size - offset - 1, size))
 237         else:
 238             handler.send_response(206)
 239             handler.send_header('Transfer-Encoding', 'chunked')
 240         handler.send_header('Content-Type', mime)
 241         handler.end_headers()
 242
 243         logger.info('[%s] Start sending "%s" to %s' %
 244                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 245         start = time.time()
 246         count = 0
 247
 248         if valid:
 249             if compatible:
 250                 if faking and not offset:
 251                     handler.wfile.write(thead)
 252                 logger.debug('"%s" is tivo compatible' % fname)
 253                 f = open(fname, 'rb')
 254                 try:
 255                     if mime == 'video/mp4':
 256                         count = qtfaststart.process(f, handler.wfile, offset)
 257                     else:
 258                         if offset:
 259                             offset -= len(thead)
 260                             f.seek(offset)
 261                         while True:
 262                             block = f.read(512 * 1024)
 263                             if not block:
 264                                 break
 265                             handler.wfile.write(block)
 266                             count += len(block)
 267                 except Exception, msg:
 268                     logger.info(msg)
 269                 f.close()
 270             else:
 271                 logger.debug('"%s" is not tivo compatible' % fname)
 272                 if offset:
 273                     count = transcode.resume_transfer(path, handler.wfile,
 274                                                       offset)
 275                 else:
 276                     count = transcode.transcode(False, path, handler.wfile,
 277                                                 tsn, mime, thead)
 278         try:
 279             if not compatible:
 280                  handler.wfile.write('0\r\n\r\n')
 281             handler.wfile.flush()
 282         except Exception, msg:
 283             logger.info(msg)
 284
 285         mega_elapsed = (time.time() - start) * 1024 * 1024
 286         if mega_elapsed < 1:
 287             mega_elapsed = 1
 288         rate = count * 8.0 / mega_elapsed
 289         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 290                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 291                      tivo_name, count, rate))
 292
 293         if fname.endswith('.pyTivo-temp'):
 294             os.remove(fname)
 295
 296     def __duration(self, full_path):
 297         return transcode.video_info(full_path)['millisecs']
 298
 299     def __total_items(self, full_path):
 300         count = 0
 301         try:
 302             full_path = unicode(full_path, 'utf-8')
 303             for f in os.listdir(full_path):
 304                 if f.startswith('.'):
 305                     continue
 306                 f = os.path.join(full_path, f)
 307                 f2 = f.encode('utf-8')
 308                 if os.path.isdir(f):
 309                     count += 1
 310                 elif use_extensions:
 311                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 312                         count += 1
 313                 elif f2 in transcode.info_cache:
 314                     if transcode.supported_format(f2):
 315                         count += 1
 316         except:
 317             pass
 318         return count
 319
 320     def __est_size(self, full_path, tsn='', mime=''):
 321         # Size is estimated by taking audio and video bit rate adding 2%
 322
 323         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 324             return os.path.getsize(unicode(full_path, 'utf-8'))
 325         else:
 326             # Must be re-encoded
 327             audioBPS = config.getMaxAudioBR(tsn) * 1000
 328             #audioBPS = config.strtod(config.getAudioBR(tsn))
 329             videoBPS = transcode.select_videostr(full_path, tsn)
 330             bitrate =  audioBPS + videoBPS
 331             return int((self.__duration(full_path) / 1000) *
 332                        (bitrate * 1.02 / 8))
 333
 334     def metadata_full(self, full_path, tsn='', mime='', mtime=None):
 335         data = {}
 336         vInfo = transcode.video_info(full_path)
 337
 338         if ((int(vInfo['vHeight']) >= 720 and
 339              config.getTivoHeight >= 720) or
 340             (int(vInfo['vWidth']) >= 1280 and
 341              config.getTivoWidth >= 1280)):
 342             data['showingBits'] = '4096'
 343
 344         data.update(metadata.basic(full_path, mtime))
 345         if full_path[-5:].lower() == '.tivo':
 346             data.update(metadata.from_tivo(full_path))
 347         if full_path[-4:].lower() == '.wtv':
 348             data.update(metadata.from_mscore(vInfo['rawmeta']))
 349
 350         if 'episodeNumber' in data:
 351             try:
 352                 ep = int(data['episodeNumber'])
 353             except:
 354                 ep = 0
 355             data['episodeNumber'] = str(ep)
 356
 357         if config.getDebug() and 'vHost' not in data:
 358             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 359             if compatible:
 360                 transcode_options = {}
 361             else:
 362                 transcode_options = transcode.transcode(True, full_path,
 363                                                         '', tsn, mime)
 364             data['vHost'] = (
 365                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 366                 ['SOURCE INFO: '] +
 367                 ["%s=%s" % (k, v)
 368                  for k, v in sorted(vInfo.items(), reverse=True)] +
 369                 ['TRANSCODE OPTIONS: '] +
 370                 ["%s" % (v) for k, v in transcode_options.items()] +
 371                 ['SOURCE FILE: ', os.path.basename(full_path)]
 372             )
 373
 374         now = datetime.utcnow()
 375         if 'time' in data:
 376             if data['time'].lower() == 'file':
 377                 if not mtime:
 378                     mtime = os.path.getmtime(unicode(full_path, 'utf-8'))
 379                 try:
 380                     now = datetime.utcfromtimestamp(mtime)
 381                 except:
 382                     logger.warning('Bad file time on ' + full_path)
 383             elif data['time'].lower() == 'oad':
 384                     now = isodt(data['originalAirDate'])
 385             else:
 386                 try:
 387                     now = isodt(data['time'])
 388                 except:
 389                     logger.warning('Bad time format: ' + data['time'] +
 390                                    ' , using current time')
 391
 392         duration = self.__duration(full_path)
 393         duration_delta = timedelta(milliseconds = duration)
 394         min = duration_delta.seconds / 60
 395         sec = duration_delta.seconds % 60
 396         hours = min / 60
 397         min = min % 60
 398
 399         data.update({'time': now.isoformat(),
 400                      'startTime': now.isoformat(),
 401                      'stopTime': (now + duration_delta).isoformat(),
 402                      'size': self.__est_size(full_path, tsn, mime),
 403                      'duration': duration,
 404                      'iso_duration': ('P%sDT%sH%sM%sS' %
 405                           (duration_delta.days, hours, min, sec))})
 406
 407         return data
 408
 409     def QueryContainer(self, handler, query):
 410         tsn = handler.headers.getheader('tsn', '')
 411         subcname = query['Container'][0]
 412         useragent = handler.headers.getheader('User-Agent', '')
 413
 414         if not self.get_local_path(handler, query):
 415             handler.send_error(404)
 416             return
 417
 418         container = handler.container
 419         force_alpha = container.getboolean('force_alpha')
 420         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 421
 422         files, total, start = self.get_files(handler, query,
 423                                              self.video_file_filter,
 424                                              force_alpha)
 425
 426         videos = []
 427         local_base_path = self.get_local_base_path(handler, query)
 428         for f in files:
 429             video = VideoDetails()
 430             mtime = f.mdate
 431             try:
 432                 ltime = time.localtime(mtime)
 433             except:
 434                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 435                 mtime = time.time()
 436                 ltime = time.localtime(mtime)
 437             video['captureDate'] = hex(int(mtime))
 438             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 439             video['name'] = os.path.basename(f.name)
 440             video['path'] = f.name
 441             video['part_path'] = f.name.replace(local_base_path, '', 1)
 442             if not video['part_path'].startswith(os.path.sep):
 443                 video['part_path'] = os.path.sep + video['part_path']
 444             video['title'] = os.path.basename(f.name)
 445             video['is_dir'] = f.isdir
 446             if video['is_dir']:
 447                 video['small_path'] = subcname + '/' + video['name']
 448                 video['total_items'] = self.__total_items(f.name)
 449             else:
 450                 if len(files) == 1 or f.name in transcode.info_cache:
 451                     video['valid'] = transcode.supported_format(f.name)
 452                     if video['valid']:
 453                         video.update(self.metadata_full(f.name, tsn,
 454                                      mtime=mtime))
 455                         if len(files) == 1:
 456                             video['captureDate'] = hex(isogm(video['time']))
 457                 else:
 458                     video['valid'] = True
 459                     video.update(metadata.basic(f.name, mtime))
 460
 461                 if self.use_ts(tsn, f.name):
 462                     video['mime'] = 'video/x-tivo-mpeg-ts'
 463                 else:
 464                     video['mime'] = 'video/x-tivo-mpeg'
 465
 466                 video['textSize'] = metadata.human_size(f.size)
 467
 468             videos.append(video)
 469
 470         logger.debug('mobileagent: %d useragent: %s' % (useragent.lower().find('mobile'), useragent.lower()))
 471         use_mobile = useragent.lower().find('mobile') > 0
 472         if use_html:
 473             if use_mobile:
 474                 t = Template(HTML_CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 475             else:
 476                 t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 477         else:
 478             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 479
 480         t.container = handler.cname
 481         t.name = subcname
 482         t.total = total
 483         t.start = start
 484         t.videos = videos
 485         t.quote = quote
 486         t.escape = escape
 487         t.crc = zlib.crc32
 488         t.guid = config.getGUID()
 489         t.tivos = config.tivos
 490         if use_html:
 491             handler.send_html(str(t))
 492         else:
 493             handler.send_xml(str(t))
 494
 495     def use_ts(self, tsn, file_path):
 496         if config.is_ts_capable(tsn):
 497             if file_path[-5:].lower() == '.tivo':
 498                 try:
 499                     flag = file(file_path).read(8)
 500                 except:
 501                     return False
 502                 if ord(flag[7]) & 0x20:
 503                     return True
 504             elif config.has_ts_flag():
 505                 return True
 506
 507         return False
 508
 509     def get_details_xml(self, tsn, file_path):
 510         if (tsn, file_path) in self.tvbus_cache:
 511             details = self.tvbus_cache[(tsn, file_path)]
 512         else:
 513             file_info = VideoDetails()
 514             file_info['valid'] = transcode.supported_format(file_path)
 515             if file_info['valid']:
 516                 file_info.update(self.metadata_full(file_path, tsn))
 517
 518             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 519             t.video = file_info
 520             t.escape = escape
 521             t.get_tv = metadata.get_tv
 522             t.get_mpaa = metadata.get_mpaa
 523             t.get_stars = metadata.get_stars
 524             details = str(t)
 525             self.tvbus_cache[(tsn, file_path)] = details
 526         return details
 527
 528     def tivo_header(self, tsn, path, mime):
 529         def pad(length, align):
 530             extra = length % align
 531             if extra:
 532                 extra = align - extra
 533             return extra
 534
 535         if mime == 'video/x-tivo-mpeg-ts':
 536             flag = 45
 537         else:
 538             flag = 13
 539         details = self.get_details_xml(tsn, path)
 540         ld = len(details)
 541         chunk = details + '\0' * (pad(ld, 4) + 4)
 542         lc = len(chunk)
 543         blocklen = lc * 2 + 40
 544         padding = pad(blocklen, 1024)
 545
 546         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 547                                             padding + blocklen, 2),
 548                         struct.pack('>LLHH', lc + 12, ld, 1, 0),
 549                         chunk,
 550                         struct.pack('>LLHH', lc + 12, ld, 2, 0),
 551                         chunk, '\0' * padding])
 552
 553     def TVBusQuery(self, handler, query):
 554         tsn = handler.headers.getheader('tsn', '')
 555         f = query['File'][0]
 556         path = self.get_local_path(handler, query)
 557         file_path = os.path.normpath(path + '/' + f)
 558
 559         details = self.get_details_xml(tsn, file_path)
 560
 561         handler.send_xml(details)
 562
 563 class Video(BaseVideo, Pushable):
 564         pass
 565
 566 class VideoDetails(DictMixin):
 567
 568     def __init__(self, d=None):
 569         if d:
 570             self.d = d
 571         else:
 572             self.d = {}
 573
 574     def __getitem__(self, key):
 575         if key not in self.d:
 576             self.d[key] = self.default(key)
 577         return self.d[key]
 578
 579     def __contains__(self, key):
 580         return True
 581
 582     def __setitem__(self, key, value):
 583         self.d[key] = value
 584
 585     def __delitem__(self):
 586         del self.d[key]
 587
 588     def keys(self):
 589         return self.d.keys()
 590
 591     def __iter__(self):
 592         return self.d.__iter__()
 593
 594     def iteritems(self):
 595         return self.d.iteritems()
 596
 597     def default(self, key):
 598         defaults = {
 599             'showingBits' : '0',
 600             'displayMajorNumber' : '0',
 601             'displayMinorNumber' : '0',
 602             'isEpisode' : 'true',
 603             'colorCode' : ('COLOR', '4'),
 604             'showType' : ('SERIES', '5')
 605         }
 606         if key in defaults:
 607             return defaults[key]
 608         elif key.startswith('v'):
 609             return []
 610         else:
 611             return ''