plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 from operator import itemgetter
  25
  26 logger = logging.getLogger('pyTivo.video.video')
  27
  28 SCRIPTDIR = os.path.dirname(__file__)
  29
  30 CLASS_NAME = 'Video'
  31
  32 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  33
  34 # Preload the templates
  35 def tmpl(name):
  36     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  37
  38 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  39 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  40 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  41
  42 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  43 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  44 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  45 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  46 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  47 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  48 .wm .wmd .wtv .yuv""".split()
  49
  50 use_extensions = True
  51 try:
  52     assert(config.get_bin('ffmpeg'))
  53 except:
  54     use_extensions = False
  55
  56 queue = []  # Recordings to push
  57
  58 def uniso(iso):
  59     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  60
  61 def isodt(iso):
  62     return datetime(*uniso(iso)[:6])
  63
  64 def isogm(iso):
  65     return int(calendar.timegm(uniso(iso)))
  66
  67 class Pushable(object):
  68
  69     def push_one_file(self, f):
  70         file_info = VideoDetails()
  71         file_info['valid'] = transcode.supported_format(f['path'])
  72
  73         temp_share = config.get_server('temp_share', '')
  74         temp_share_path = ''
  75         if temp_share:
  76             for name, data in config.getShares():
  77                 if temp_share == name:
  78                     temp_share_path = data.get('path')
  79
  80         mime = 'video/mpeg'
  81         if config.isHDtivo(f['tsn']):
  82             for m in ['video/mp4', 'video/bif']:
  83                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  84                     mime = m
  85                     break
  86
  87             if (mime == 'video/mpeg' and
  88                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  89                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'], temp_share_path)
  90                 if new_path:
  91                     mime = 'video/mp4'
  92                     f['name'] = new_path
  93                     if temp_share_path:
  94                         ip = config.get_ip()
  95                         port = config.getPort()
  96                         container = quote(temp_share) + '/'
  97                         f['url'] = 'http://%s:%s/%s' % (ip, port, container)
  98
  99         if file_info['valid']:
 100             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
 101
 102         url = f['url'] + quote(f['name'])
 103
 104         title = file_info['seriesTitle']
 105         if not title:
 106             title = file_info['title']
 107
 108         source = file_info['seriesId']
 109         if not source:
 110             source = title
 111
 112         subtitle = file_info['episodeTitle']
 113         try:
 114             m = mind.getMind(f['tsn'])
 115             m.pushVideo(
 116                 tsn = f['tsn'],
 117                 url = url,
 118                 description = file_info['description'],
 119                 duration = file_info['duration'] / 1000,
 120                 size = file_info['size'],
 121                 title = title,
 122                 subtitle = subtitle,
 123                 source = source,
 124                 mime = mime,
 125                 tvrating = file_info['tvRating'])
 126         except Exception, msg:
 127             logger.error(msg)
 128
 129     def process_queue(self):
 130         while queue:
 131             time.sleep(5)
 132             item = queue.pop(0)
 133             self.push_one_file(item)
 134
 135     def readip(self):
 136         """ returns your external IP address by querying dyndns.org """
 137         f = urllib.urlopen('http://checkip.dyndns.org/')
 138         s = f.read()
 139         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 140         return m.group(0)
 141
 142     def Push(self, handler, query):
 143         try:
 144             tsn = query['tsn'][0]
 145         except:
 146             logger.error('Push requires a TiVo Service Number')
 147             handler.send_error(404)
 148             return
 149
 150         if not tsn in config.tivos:
 151             for key, value in config.tivos.items():
 152                 if value.get('name') == tsn:
 153                     tsn = key
 154                     break
 155         try:
 156             tivo_name = config.tivos[tsn]['name']
 157         except:
 158             tivo_name = tsn
 159
 160         container = quote(query['Container'][0].split('/')[0])
 161         ip = config.get_ip(tsn)
 162         port = config.getPort()
 163
 164         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 165         if config.getIsExternal(tsn):
 166             exturl = config.get_server('externalurl')
 167             if exturl:
 168                 if not exturl.endswith('/'):
 169                     exturl += '/'
 170                 baseurl = exturl + container
 171             else:
 172                 ip = self.readip()
 173                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 174
 175         path = self.get_local_base_path(handler, query)
 176
 177         files = query.get('File', [])
 178         for f in files:
 179             file_path = os.path.normpath(path + '/' + f)
 180             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 181                           'url': baseurl})
 182             if len(queue) == 1:
 183                 thread.start_new_thread(Video.process_queue, (self,))
 184
 185             logger.info('[%s] Queued "%s" for Push to %s' %
 186                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 187                          unicode(file_path, 'utf-8'), tivo_name))
 188
 189         files = [unicode(f, 'utf-8') for f in files]
 190         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 191
 192 class BaseVideo(Plugin):
 193
 194     CONTENT_TYPE = 'x-container/tivo-videos'
 195
 196     tvbus_cache = LRUCache(1)
 197
 198     def video_file_filter(self, full_path, type=None):
 199         if os.path.isdir(unicode(full_path, 'utf-8')):
 200             return True
 201         if use_extensions:
 202             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 203         else:
 204             return transcode.supported_format(full_path)
 205
 206     def send_file(self, handler, path, query):
 207         mime = 'video/x-tivo-mpeg'
 208         tsn = handler.headers.getheader('tsn', '')
 209         try:
 210             assert(tsn)
 211             tivo_name = config.tivos[tsn].get('name', tsn)
 212         except:
 213             tivo_name = handler.address_string()
 214
 215         is_tivo_file = (path[-5:].lower() == '.tivo')
 216
 217         if 'Format' in query:
 218             mime = query['Format'][0]
 219
 220         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 221         compatible = (not needs_tivodecode and
 222                       transcode.tivo_compatible(path, tsn, mime)[0])
 223
 224         try:  # "bytes=XXX-"
 225             offset = int(handler.headers.getheader('Range')[6:-1])
 226         except:
 227             offset = 0
 228
 229         if needs_tivodecode:
 230             valid = bool(config.get_bin('tivodecode') and
 231                          config.get_server('tivo_mak'))
 232         else:
 233             valid = True
 234
 235         if valid and offset:
 236             valid = ((compatible and offset < os.path.getsize(path)) or
 237                      (not compatible and transcode.is_resumable(path, offset)))
 238
 239         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 240         faking = (mime == 'video/x-tivo-mpeg' and
 241                   not (is_tivo_file and compatible))
 242         fname = unicode(path, 'utf-8')
 243         thead = ''
 244         if faking:
 245             thead = self.tivo_header(tsn, path, mime)
 246         if compatible:
 247             size = os.path.getsize(fname) + len(thead)
 248             handler.send_response(200)
 249             handler.send_header('Content-Length', size - offset)
 250             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 251                                 (offset, size - offset - 1, size))
 252         else:
 253             handler.send_response(206)
 254             handler.send_header('Transfer-Encoding', 'chunked')
 255         handler.send_header('Content-Type', mime)
 256         handler.end_headers()
 257
 258         logger.info('[%s] Start sending "%s" to %s' %
 259                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 260         start = time.time()
 261         count = 0
 262
 263         if valid:
 264             if compatible:
 265                 if faking and not offset:
 266                     handler.wfile.write(thead)
 267                 logger.debug('"%s" is tivo compatible' % fname)
 268                 f = open(fname, 'rb')
 269                 try:
 270                     if mime == 'video/mp4':
 271                         count = qtfaststart.process(f, handler.wfile, offset)
 272                     else:
 273                         if offset:
 274                             offset -= len(thead)
 275                             f.seek(offset)
 276                         while True:
 277                             block = f.read(512 * 1024)
 278                             if not block:
 279                                 break
 280                             handler.wfile.write(block)
 281                             count += len(block)
 282                 except Exception, msg:
 283                     logger.info(msg)
 284                 f.close()
 285             else:
 286                 logger.debug('"%s" is not tivo compatible' % fname)
 287                 if offset:
 288                     count = transcode.resume_transfer(path, handler.wfile,
 289                                                       offset)
 290                 else:
 291                     count = transcode.transcode(False, path, handler.wfile,
 292                                                 tsn, mime, thead)
 293         try:
 294             if not compatible:
 295                  handler.wfile.write('0\r\n\r\n')
 296             handler.wfile.flush()
 297         except Exception, msg:
 298             logger.info(msg)
 299
 300         mega_elapsed = (time.time() - start) * 1024 * 1024
 301         if mega_elapsed < 1:
 302             mega_elapsed = 1
 303         rate = count * 8.0 / mega_elapsed
 304         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 305                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 306                      tivo_name, count, rate))
 307
 308         if fname.endswith('.pyTivo-temp'):
 309             os.remove(fname)
 310
 311     def __duration(self, full_path):
 312         return transcode.video_info(full_path)['millisecs']
 313
 314     def __total_items(self, full_path):
 315         count = 0
 316         try:
 317             full_path = unicode(full_path, 'utf-8')
 318             for f in os.listdir(full_path):
 319                 if f.startswith('.'):
 320                     continue
 321                 f = os.path.join(full_path, f)
 322                 f2 = f.encode('utf-8')
 323                 if os.path.isdir(f):
 324                     count += 1
 325                 elif use_extensions:
 326                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 327                         count += 1
 328                 elif f2 in transcode.info_cache:
 329                     if transcode.supported_format(f2):
 330                         count += 1
 331         except:
 332             pass
 333         return count
 334
 335     def __est_size(self, full_path, tsn='', mime=''):
 336         # Size is estimated by taking audio and video bit rate adding 2%
 337
 338         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 339             return os.path.getsize(unicode(full_path, 'utf-8'))
 340         else:
 341             # Must be re-encoded
 342             audioBPS = config.getMaxAudioBR(tsn) * 1000
 343             #audioBPS = config.strtod(config.getAudioBR(tsn))
 344             videoBPS = transcode.select_videostr(full_path, tsn)
 345             bitrate =  audioBPS + videoBPS
 346             return int((self.__duration(full_path) / 1000) *
 347                        (bitrate * 1.02 / 8))
 348
 349     def metadata_full(self, full_path, tsn='', mime='', mtime=None):
 350         data = {}
 351         vInfo = transcode.video_info(full_path)
 352
 353         if ((int(vInfo['vHeight']) >= 720 and
 354              config.getTivoHeight >= 720) or
 355             (int(vInfo['vWidth']) >= 1280 and
 356              config.getTivoWidth >= 1280)):
 357             data['showingBits'] = '4096'
 358
 359         data.update(metadata.basic(full_path, mtime))
 360         if full_path[-5:].lower() == '.tivo':
 361             data.update(metadata.from_tivo(full_path))
 362         if full_path[-4:].lower() == '.wtv':
 363             data.update(metadata.from_mscore(vInfo['rawmeta']))
 364
 365         if 'episodeNumber' in data:
 366             try:
 367                 ep = int(data['episodeNumber'])
 368             except:
 369                 ep = 0
 370             data['episodeNumber'] = str(ep)
 371
 372         if config.getDebug() and 'vHost' not in data:
 373             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 374             if compatible:
 375                 transcode_options = {}
 376             else:
 377                 transcode_options = transcode.transcode(True, full_path,
 378                                                         '', tsn, mime)
 379             data['vHost'] = (
 380                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 381                 ['SOURCE INFO: '] +
 382                 ["%s=%s" % (k, v)
 383                  for k, v in sorted(vInfo.items(), reverse=True)] +
 384                 ['TRANSCODE OPTIONS: '] +
 385                 ["%s" % (v) for k, v in transcode_options.items()] +
 386                 ['SOURCE FILE: ', os.path.basename(full_path)]
 387             )
 388
 389         now = datetime.utcnow()
 390         if 'time' in data:
 391             if data['time'].lower() == 'file':
 392                 if not mtime:
 393                     mtime = os.path.getmtime(unicode(full_path, 'utf-8'))
 394                 try:
 395                     now = datetime.utcfromtimestamp(mtime)
 396                 except:
 397                     logger.warning('Bad file time on ' + full_path)
 398             elif data['time'].lower() == 'oad':
 399                     now = isodt(data['originalAirDate'])
 400             else:
 401                 try:
 402                     now = isodt(data['time'])
 403                 except:
 404                     logger.warning('Bad time format: "' + data['time'] +
 405                                    '", using current time')
 406
 407         duration = self.__duration(full_path)
 408         duration_delta = timedelta(milliseconds = duration)
 409         min = duration_delta.seconds / 60
 410         sec = duration_delta.seconds % 60
 411         hours = min / 60
 412         min = min % 60
 413
 414         data.update({'time': now.isoformat(),
 415                      'startTime': now.isoformat(),
 416                      'stopTime': (now + duration_delta).isoformat(),
 417                      'size': self.__est_size(full_path, tsn, mime),
 418                      'duration': duration,
 419                      'iso_duration': ('P%sDT%sH%sM%sS' %
 420                           (duration_delta.days, hours, min, sec))})
 421
 422         return data
 423
 424     def QueryContainer(self, handler, query):
 425         tsn = handler.headers.getheader('tsn', '')
 426         subcname = query['Container'][0]
 427         useragent = handler.headers.getheader('User-Agent', '')
 428
 429         if not self.get_local_path(handler, query):
 430             handler.send_error(404)
 431             return
 432
 433         container = handler.container
 434         force_alpha = container.getboolean('force_alpha')
 435         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 436
 437         files, total, start = self.get_files(handler, query,
 438                                              self.video_file_filter,
 439                                              force_alpha)
 440
 441         videos = []
 442         local_base_path = self.get_local_base_path(handler, query)
 443         resort = False
 444         for f in files:
 445             video = VideoDetails()
 446             mtime = f.mdate
 447             try:
 448                 ltime = time.localtime(mtime)
 449             except:
 450                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 451                 mtime = time.time()
 452                 ltime = time.localtime(mtime)
 453             video['captureDate'] = hex(int(mtime))
 454             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 455             video['name'] = os.path.basename(f.name)
 456             video['path'] = f.name
 457             video['part_path'] = f.name.replace(local_base_path, '', 1)
 458             if not video['part_path'].startswith(os.path.sep):
 459                 video['part_path'] = os.path.sep + video['part_path']
 460             video['title'] = os.path.basename(f.name)
 461             video['is_dir'] = f.isdir
 462             if video['is_dir']:
 463                 video['small_path'] = subcname + '/' + video['name']
 464                 video['total_items'] = self.__total_items(f.name)
 465             else:
 466                 if len(files) == 1 or f.name in transcode.info_cache:
 467                     video['valid'] = transcode.supported_format(f.name)
 468                     if video['valid']:
 469                         video.update(self.metadata_full(f.name, tsn,
 470                                      mtime=mtime))
 471                 else:
 472                     video['valid'] = True
 473                     video.update(metadata.basic(f.name, mtime))
 474
 475                 if 'time' in video and video['time'] != '':
 476                     if video['time'].lower() == 'oad':
 477                         video['time'] = video['originalAirDate']
 478                         resort = True
 479                     try:
 480                         video['captureDate'] = hex(isogm(video['time']))
 481                         video['textDate'] = time.strftime('%b %d, %Y', time.localtime(isogm(video['time'])))
 482                         resort = True
 483                     except:
 484                         logger.warning('Bad time format: "' + video['time'] +
 485                                        '", using current time')
 486
 487                 if self.use_ts(tsn, f.name):
 488                     video['mime'] = 'video/x-tivo-mpeg-ts'
 489                 else:
 490                     video['mime'] = 'video/x-tivo-mpeg'
 491
 492                 video['textSize'] = metadata.human_size(f.size)
 493
 494             videos.append(video)
 495
 496         if use_html:
 497             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 498         else:
 499             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 500
 501         sortby = query.get('SortOrder', ['Normal'])[0].lower()
 502         t.sortby = sortby
 503         if use_html and resort:
 504             if sortby == 'capturedate':
 505                 logger.info('re-sorting by captureDate, reverse=True')
 506                 videos.sort(key=itemgetter('captureDate'), reverse=True)
 507             elif sortby == '!capturedate':
 508                 logger.info('re-sorting by captureDate, reverse=False')
 509                 videos.sort(key=itemgetter('captureDate'), reverse=False)
 510
 511         t.container = handler.cname
 512         t.name = subcname
 513         t.total = total
 514         t.start = start
 515         t.videos = videos
 516         t.quote = quote
 517         t.escape = escape
 518         t.crc = zlib.crc32
 519         t.guid = config.getGUID()
 520         t.tivos = config.tivos
 521         if use_html:
 522             handler.send_html(str(t))
 523         else:
 524             handler.send_xml(str(t))
 525
 526     def use_ts(self, tsn, file_path):
 527         if config.is_ts_capable(tsn):
 528             if file_path[-5:].lower() == '.tivo':
 529                 try:
 530                     flag = file(file_path).read(8)
 531                 except:
 532                     return False
 533                 if ord(flag[7]) & 0x20:
 534                     return True
 535             elif config.has_ts_flag():
 536                 return True
 537
 538         return False
 539
 540     def get_details_xml(self, tsn, file_path):
 541         if (tsn, file_path) in self.tvbus_cache:
 542             details = self.tvbus_cache[(tsn, file_path)]
 543         else:
 544             file_info = VideoDetails()
 545             file_info['valid'] = transcode.supported_format(file_path)
 546             if file_info['valid']:
 547                 file_info.update(self.metadata_full(file_path, tsn))
 548
 549             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 550             t.video = file_info
 551             t.escape = escape
 552             t.get_tv = metadata.get_tv
 553             t.get_mpaa = metadata.get_mpaa
 554             t.get_stars = metadata.get_stars
 555             details = str(t)
 556             self.tvbus_cache[(tsn, file_path)] = details
 557         return details
 558
 559     def tivo_header(self, tsn, path, mime):
 560         def pad(length, align):
 561             extra = length % align
 562             if extra:
 563                 extra = align - extra
 564             return extra
 565
 566         if mime == 'video/x-tivo-mpeg-ts':
 567             flag = 45
 568         else:
 569             flag = 13
 570         details = self.get_details_xml(tsn, path)
 571         ld = len(details)
 572         chunk = details + '\0' * (pad(ld, 4) + 4)
 573         lc = len(chunk)
 574         blocklen = lc * 2 + 40
 575         padding = pad(blocklen, 1024)
 576
 577         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 578                                             padding + blocklen, 2),
 579                         struct.pack('>LLHH', lc + 12, ld, 1, 0),
 580                         chunk,
 581                         struct.pack('>LLHH', lc + 12, ld, 2, 0),
 582                         chunk, '\0' * padding])
 583
 584     def TVBusQuery(self, handler, query):
 585         tsn = handler.headers.getheader('tsn', '')
 586         f = query['File'][0]
 587         path = self.get_local_path(handler, query)
 588         file_path = os.path.normpath(path + '/' + f)
 589
 590         details = self.get_details_xml(tsn, file_path)
 591
 592         handler.send_xml(details)
 593
 594 class Video(BaseVideo, Pushable):
 595         pass
 596
 597 class VideoDetails(DictMixin):
 598
 599     def __init__(self, d=None):
 600         if d:
 601             self.d = d
 602         else:
 603             self.d = {}
 604
 605     def __getitem__(self, key):
 606         if key not in self.d:
 607             self.d[key] = self.default(key)
 608         return self.d[key]
 609
 610     def __contains__(self, key):
 611         return True
 612
 613     def __setitem__(self, key, value):
 614         self.d[key] = value
 615
 616     def __delitem__(self):
 617         del self.d[key]
 618
 619     def keys(self):
 620         return self.d.keys()
 621
 622     def __iter__(self):
 623         return self.d.__iter__()
 624
 625     def iteritems(self):
 626         return self.d.iteritems()
 627
 628     def default(self, key):
 629         defaults = {
 630             'showingBits' : '0',
 631             'displayMajorNumber' : '0',
 632             'displayMinorNumber' : '0',
 633             'isEpisode' : 'true',
 634             'colorCode' : ('COLOR', '4'),
 635             'showType' : ('SERIES', '5')
 636         }
 637         if key in defaults:
 638             return defaults[key]
 639         elif key.startswith('v'):
 640             return []
 641         else:
 642             return ''