plugins/video/video.py

   1 import calendar
   2 import cgi
   3 import logging
   4 import os
   5 import re
   6 import struct
   7 import thread
   8 import time
   9 import traceback
  10 import urllib
  11 import zlib
  12 from UserDict import DictMixin
  13 from datetime import datetime, timedelta
  14 from xml.sax.saxutils import escape
  15
  16 from Cheetah.Template import Template
  17 from lrucache import LRUCache
  18
  19 import config
  20 import metadata
  21 import mind
  22 import qtfaststart
  23 import transcode
  24 from plugin import EncodeUnicode, Plugin, quote
  25
  26 logger = logging.getLogger('pyTivo.video.video')
  27
  28 SCRIPTDIR = os.path.dirname(__file__)
  29
  30 CLASS_NAME = 'Video'
  31
  32 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  33
  34 # Preload the templates
  35 def tmpl(name):
  36     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  37
  38 HTML_CONTAINER_TEMPLATE_MOBILE = tmpl('container_mob.tmpl')
  39 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  40 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  41 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  42
  43 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  44 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  45 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  46 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  47 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  48 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  49 .wm .wmd .wtv .yuv""".split()
  50
  51 use_extensions = True
  52 try:
  53     assert(config.get_bin('ffmpeg'))
  54 except:
  55     use_extensions = False
  56
  57 queue = []  # Recordings to push
  58
  59 def uniso(iso):
  60     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  61
  62 def isodt(iso):
  63     return datetime(*uniso(iso)[:6])
  64
  65 def isogm(iso):
  66     return int(calendar.timegm(uniso(iso)))
  67
  68 class Pushable(object):
  69
  70     def push_one_file(self, f):
  71         file_info = VideoDetails()
  72         file_info['valid'] = transcode.supported_format(f['path'])
  73
  74         temp_share = config.get_server('temp_share', '')
  75         temp_share_path = ''
  76         if temp_share:
  77             for name, data in config.getShares():
  78                 if temp_share == name:
  79                     temp_share_path = data.get('path')
  80
  81         mime = 'video/mpeg'
  82         if config.isHDtivo(f['tsn']):
  83             for m in ['video/mp4', 'video/bif']:
  84                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  85                     mime = m
  86                     break
  87
  88             if (mime == 'video/mpeg' and
  89                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  90                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'], temp_share_path)
  91                 if new_path:
  92                     mime = 'video/mp4'
  93                     f['name'] = new_path
  94                     if temp_share_path:
  95                         ip = config.get_ip()
  96                         port = config.getPort()
  97                         container = quote(temp_share) + '/'
  98                         f['url'] = 'http://%s:%s/%s' % (ip, port, container)
  99
 100         if file_info['valid']:
 101             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
 102
 103         url = f['url'] + quote(f['name'])
 104
 105         title = file_info['seriesTitle']
 106         if not title:
 107             title = file_info['title']
 108
 109         source = file_info['seriesId']
 110         if not source:
 111             source = title
 112
 113         subtitle = file_info['episodeTitle']
 114         try:
 115             m = mind.getMind(f['tsn'])
 116             m.pushVideo(
 117                 tsn = f['tsn'],
 118                 url = url,
 119                 description = file_info['description'],
 120                 duration = file_info['duration'] / 1000,
 121                 size = file_info['size'],
 122                 title = title,
 123                 subtitle = subtitle,
 124                 source = source,
 125                 mime = mime,
 126                 tvrating = file_info['tvRating'])
 127         except Exception, msg:
 128             logger.error(msg)
 129
 130     def process_queue(self):
 131         while queue:
 132             time.sleep(5)
 133             item = queue.pop(0)
 134             self.push_one_file(item)
 135
 136     def readip(self):
 137         """ returns your external IP address by querying dyndns.org """
 138         f = urllib.urlopen('http://checkip.dyndns.org/')
 139         s = f.read()
 140         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 141         return m.group(0)
 142
 143     def Push(self, handler, query):
 144         tsn = query['tsn'][0]
 145         for key in config.tivo_names:
 146             if config.tivo_names[key] == tsn:
 147                 tsn = key
 148                 break
 149         tivo_name = config.tivo_names.get(tsn, tsn)
 150
 151         container = quote(query['Container'][0].split('/')[0])
 152         ip = config.get_ip(tsn)
 153         port = config.getPort()
 154
 155         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 156         if config.getIsExternal(tsn):
 157             exturl = config.get_server('externalurl')
 158             if exturl:
 159                 baseurl = exturl
 160             else:
 161                 ip = self.readip()
 162                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 163
 164         path = self.get_local_base_path(handler, query)
 165
 166         files = query.get('File', [])
 167         for f in files:
 168             file_path = path + os.path.normpath(f)
 169             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 170                           'url': baseurl})
 171             if len(queue) == 1:
 172                 thread.start_new_thread(Video.process_queue, (self,))
 173
 174             logger.info('[%s] Queued "%s" for Push to %s' %
 175                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 176                          unicode(file_path, 'utf-8'), tivo_name))
 177
 178         files = [unicode(f, 'utf-8') for f in files]
 179         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 180
 181 class BaseVideo(Plugin):
 182
 183     CONTENT_TYPE = 'x-container/tivo-videos'
 184
 185     tvbus_cache = LRUCache(1)
 186
 187     def pre_cache(self, full_path):
 188         if Video.video_file_filter(self, full_path):
 189             transcode.supported_format(full_path)
 190
 191     def video_file_filter(self, full_path, type=None):
 192         if os.path.isdir(unicode(full_path, 'utf-8')):
 193             return True
 194         if use_extensions:
 195             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 196         else:
 197             return transcode.supported_format(full_path)
 198
 199     def send_file(self, handler, path, query):
 200         mime = 'video/x-tivo-mpeg'
 201         tsn = handler.headers.getheader('tsn', '')
 202         tivo_name = config.tivo_names.get(tsn, tsn)
 203
 204         is_tivo_file = (path[-5:].lower() == '.tivo')
 205
 206         if 'Format' in query:
 207             mime = query['Format'][0]
 208
 209         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 210         compatible = (not needs_tivodecode and
 211                       transcode.tivo_compatible(path, tsn, mime)[0])
 212
 213         try:  # "bytes=XXX-"
 214             offset = int(handler.headers.getheader('Range')[6:-1])
 215         except:
 216             offset = 0
 217
 218         if needs_tivodecode:
 219             valid = bool(config.get_bin('tivodecode') and
 220                          config.get_server('tivo_mak'))
 221         else:
 222             valid = True
 223
 224         if valid and offset:
 225             valid = ((compatible and offset < os.stat(path).st_size) or
 226                      (not compatible and transcode.is_resumable(path, offset)))
 227
 228         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 229         faking = (mime == 'video/x-tivo-mpeg' and
 230                   not (is_tivo_file and compatible))
 231         fname = unicode(path, 'utf-8')
 232         thead = ''
 233         if faking:
 234             thead = self.tivo_header(tsn, path, mime)
 235         if compatible:
 236             size = os.stat(fname).st_size + len(thead)
 237             handler.send_response(200)
 238             handler.send_header('Content-Length', size - offset)
 239             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 240                                 (offset, size - offset - 1, size))
 241         else:
 242             handler.send_response(206)
 243             handler.send_header('Transfer-Encoding', 'chunked')
 244         handler.send_header('Content-Type', mime)
 245         handler.send_header('Connection', 'close')
 246         handler.end_headers()
 247
 248         logger.info('[%s] Start sending "%s" to %s' %
 249                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 250         start = time.time()
 251         count = 0
 252
 253         if valid:
 254             if compatible:
 255                 if faking and not offset:
 256                     handler.wfile.write(thead)
 257                 logger.debug('"%s" is tivo compatible' % fname)
 258                 f = open(fname, 'rb')
 259                 try:
 260                     if mime == 'video/mp4':
 261                         count = qtfaststart.process(f, handler.wfile, offset)
 262                     else:
 263                         if offset:
 264                             offset -= len(thead)
 265                             f.seek(offset)
 266                         while True:
 267                             block = f.read(512 * 1024)
 268                             if not block:
 269                                 break
 270                             handler.wfile.write(block)
 271                             count += len(block)
 272                 except Exception, msg:
 273                     logger.info(msg)
 274                 f.close()
 275             else:
 276                 logger.debug('"%s" is not tivo compatible' % fname)
 277                 if offset:
 278                     count = transcode.resume_transfer(path, handler.wfile,
 279                                                       offset)
 280                 else:
 281                     count = transcode.transcode(False, path, handler.wfile,
 282                                                 tsn, mime, thead)
 283         try:
 284             if not compatible:
 285                  handler.wfile.write('0\r\n\r\n')
 286             handler.wfile.flush()
 287         except Exception, msg:
 288             logger.info(msg)
 289
 290         mega_elapsed = (time.time() - start) * 1024 * 1024
 291         if mega_elapsed < 1:
 292             mega_elapsed = 1
 293         rate = count * 8.0 / mega_elapsed
 294         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 295                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 296                      tivo_name, count, rate))
 297
 298         if fname.endswith('.pyTivo-temp'):
 299             os.remove(fname)
 300
 301     def __duration(self, full_path):
 302         return transcode.video_info(full_path)['millisecs']
 303
 304     def __total_items(self, full_path):
 305         count = 0
 306         try:
 307             full_path = unicode(full_path, 'utf-8')
 308             for f in os.listdir(full_path):
 309                 if f.startswith('.'):
 310                     continue
 311                 f = os.path.join(full_path, f)
 312                 f2 = f.encode('utf-8')
 313                 if os.path.isdir(f):
 314                     count += 1
 315                 elif use_extensions:
 316                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 317                         count += 1
 318                 elif f2 in transcode.info_cache:
 319                     if transcode.supported_format(f2):
 320                         count += 1
 321         except:
 322             pass
 323         return count
 324
 325     def __est_size(self, full_path, tsn='', mime=''):
 326         # Size is estimated by taking audio and video bit rate adding 2%
 327
 328         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 329             return int(os.stat(unicode(full_path, 'utf-8')).st_size)
 330         else:
 331             # Must be re-encoded
 332             if config.get_tsn('audio_codec', tsn) == None:
 333                 audioBPS = config.getMaxAudioBR(tsn) * 1000
 334             else:
 335                 audioBPS = config.strtod(config.getAudioBR(tsn))
 336             videoBPS = transcode.select_videostr(full_path, tsn)
 337             bitrate =  audioBPS + videoBPS
 338             return int((self.__duration(full_path) / 1000) *
 339                        (bitrate * 1.02 / 8))
 340
 341     def metadata_full(self, full_path, tsn='', mime=''):
 342         data = {}
 343         vInfo = transcode.video_info(full_path)
 344
 345         if ((int(vInfo['vHeight']) >= 720 and
 346              config.getTivoHeight >= 720) or
 347             (int(vInfo['vWidth']) >= 1280 and
 348              config.getTivoWidth >= 1280)):
 349             data['showingBits'] = '4096'
 350
 351         data.update(metadata.basic(full_path))
 352         if full_path[-5:].lower() == '.tivo':
 353             data.update(metadata.from_tivo(full_path))
 354         if full_path[-4:].lower() == '.wtv':
 355             data.update(metadata.from_mscore(vInfo['rawmeta']))
 356
 357         if 'episodeNumber' in data:
 358             try:
 359                 ep = int(data['episodeNumber'])
 360             except:
 361                 ep = 0
 362             data['episodeNumber'] = str(ep)
 363
 364         if config.getDebug() and 'vHost' not in data:
 365             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 366             if compatible:
 367                 transcode_options = {}
 368             else:
 369                 transcode_options = transcode.transcode(True, full_path,
 370                                                         '', tsn, mime)
 371             data['vHost'] = (
 372                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 373                 ['SOURCE INFO: '] +
 374                 ["%s=%s" % (k, v)
 375                  for k, v in sorted(vInfo.items(), reverse=True)] +
 376                 ['TRANSCODE OPTIONS: '] +
 377                 ["%s" % (v) for k, v in transcode_options.items()] +
 378                 ['SOURCE FILE: ', os.path.basename(full_path)]
 379             )
 380
 381         now = datetime.utcnow()
 382         if 'time' in data:
 383             if data['time'].lower() == 'file':
 384                 mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 385                 if (mtime < 0):
 386                     mtime = 0
 387                 try:
 388                     now = datetime.utcfromtimestamp(mtime)
 389                 except:
 390                     logger.warning('Bad file time on ' + full_path)
 391             elif data['time'].lower() == 'oad':
 392                     now = isodt(data['originalAirDate'])
 393             else:
 394                 try:
 395                     now = isodt(data['time'])
 396                 except:
 397                     logger.warning('Bad time format: ' + data['time'] +
 398                                    ' , using current time')
 399
 400         duration = self.__duration(full_path)
 401         duration_delta = timedelta(milliseconds = duration)
 402         min = duration_delta.seconds / 60
 403         sec = duration_delta.seconds % 60
 404         hours = min / 60
 405         min = min % 60
 406
 407         data.update({'time': now.isoformat(),
 408                      'startTime': now.isoformat(),
 409                      'stopTime': (now + duration_delta).isoformat(),
 410                      'size': self.__est_size(full_path, tsn, mime),
 411                      'duration': duration,
 412                      'iso_duration': ('P%sDT%sH%sM%sS' %
 413                           (duration_delta.days, hours, min, sec))})
 414
 415         return data
 416
 417     def QueryContainer(self, handler, query):
 418         tsn = handler.headers.getheader('tsn', '')
 419         subcname = query['Container'][0]
 420         useragent = handler.headers.getheader('User-Agent', '')
 421
 422         if not self.get_local_path(handler, query):
 423             handler.send_error(404)
 424             return
 425
 426         container = handler.container
 427         precache = container.get('precache', 'False').lower() == 'true'
 428         force_alpha = container.get('force_alpha', 'False').lower() == 'true'
 429         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 430
 431         files, total, start = self.get_files(handler, query,
 432                                              self.video_file_filter,
 433                                              force_alpha)
 434
 435         videos = []
 436         local_base_path = self.get_local_base_path(handler, query)
 437         for f in files:
 438             video = VideoDetails()
 439             mtime = f.mdate
 440             try:
 441                 ltime = time.localtime(mtime)
 442             except:
 443                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 444                 mtime = int(time.time())
 445                 ltime = time.localtime(mtime)
 446             video['captureDate'] = hex(mtime)
 447             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 448             video['name'] = os.path.basename(f.name)
 449             video['path'] = f.name
 450             video['part_path'] = f.name.replace(local_base_path, '', 1)
 451             if not video['part_path'].startswith(os.path.sep):
 452                 video['part_path'] = os.path.sep + video['part_path']
 453             video['title'] = os.path.basename(f.name)
 454             video['is_dir'] = f.isdir
 455             if video['is_dir']:
 456                 video['small_path'] = subcname + '/' + video['name']
 457                 video['total_items'] = self.__total_items(f.name)
 458             else:
 459                 if precache or len(files) == 1 or f.name in transcode.info_cache:
 460                     video['valid'] = transcode.supported_format(f.name)
 461                     if video['valid']:
 462                         video.update(self.metadata_full(f.name, tsn))
 463                         if len(files) == 1:
 464                             video['captureDate'] = hex(isogm(video['time']))
 465                 else:
 466                     video['valid'] = True
 467                     video.update(metadata.basic(f.name))
 468
 469                 if config.hasTStivo(tsn):
 470                     video['mime'] = 'video/x-tivo-mpeg-ts'
 471                 else:
 472                     video['mime'] = 'video/x-tivo-mpeg'
 473
 474                 video['textSize'] = ( '%.3f GB' %
 475                     (float(f.size) / (1024 ** 3)) )
 476
 477             videos.append(video)
 478
 479         logger.debug('mobileagent: %d useragent: %s' % (useragent.lower().find('mobile'), useragent.lower()))
 480         use_mobile = useragent.lower().find('mobile') > 0
 481         if use_html:
 482             if use_mobile:
 483                 t = Template(HTML_CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 484             else:
 485                 t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 486         else:
 487             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 488
 489         t.container = handler.cname
 490         t.name = subcname
 491         t.total = total
 492         t.start = start
 493         t.videos = videos
 494         t.quote = quote
 495         t.escape = escape
 496         t.crc = zlib.crc32
 497         t.guid = config.getGUID()
 498         t.tivos = config.tivos
 499         t.tivo_names = config.tivo_names
 500         if use_html:
 501             handler.send_html(str(t))
 502         else:
 503             handler.send_xml(str(t))
 504
 505     def get_details_xml(self, tsn, file_path):
 506         if (tsn, file_path) in self.tvbus_cache:
 507             details = self.tvbus_cache[(tsn, file_path)]
 508         else:
 509             file_info = VideoDetails()
 510             file_info['valid'] = transcode.supported_format(file_path)
 511             if file_info['valid']:
 512                 file_info.update(self.metadata_full(file_path, tsn))
 513
 514             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 515             t.video = file_info
 516             t.escape = escape
 517             t.get_tv = metadata.get_tv
 518             t.get_mpaa = metadata.get_mpaa
 519             t.get_stars = metadata.get_stars
 520             details = str(t)
 521             self.tvbus_cache[(tsn, file_path)] = details
 522         return details
 523
 524     def tivo_header(self, tsn, path, mime):
 525         if mime == 'video/x-tivo-mpeg-ts':
 526             flag = 45
 527         else:
 528             flag = 13
 529         details = self.get_details_xml(tsn, path)
 530         ld = len(details)
 531         chunklen = ld * 2 + 44
 532         padding = 2048 - chunklen % 1024
 533
 534         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 535                                             padding + chunklen, 2),
 536                         struct.pack('>LLHH', ld + 16, ld, 1, 0),
 537                         details, '\0' * 4,
 538                         struct.pack('>LLHH', ld + 19, ld, 2, 0),
 539                         details, '\0' * padding])
 540
 541     def TVBusQuery(self, handler, query):
 542         tsn = handler.headers.getheader('tsn', '')
 543         f = query['File'][0]
 544         path = self.get_local_path(handler, query)
 545         file_path = path + os.path.normpath(f)
 546
 547         details = self.get_details_xml(tsn, file_path)
 548
 549         handler.send_xml(details)
 550
 551 class Video(BaseVideo, Pushable):
 552         pass
 553
 554 class VideoDetails(DictMixin):
 555
 556     def __init__(self, d=None):
 557         if d:
 558             self.d = d
 559         else:
 560             self.d = {}
 561
 562     def __getitem__(self, key):
 563         if key not in self.d:
 564             self.d[key] = self.default(key)
 565         return self.d[key]
 566
 567     def __contains__(self, key):
 568         return True
 569
 570     def __setitem__(self, key, value):
 571         self.d[key] = value
 572
 573     def __delitem__(self):
 574         del self.d[key]
 575
 576     def keys(self):
 577         return self.d.keys()
 578
 579     def __iter__(self):
 580         return self.d.__iter__()
 581
 582     def iteritems(self):
 583         return self.d.iteritems()
 584
 585     def default(self, key):
 586         defaults = {
 587             'showingBits' : '0',
 588             'displayMajorNumber' : '0',
 589             'displayMinorNumber' : '0',
 590             'isEpisode' : 'true',
 591             'colorCode' : ('COLOR', '4'),
 592             'showType' : ('SERIES', '5')
 593         }
 594         if key in defaults:
 595             return defaults[key]
 596         elif key.startswith('v'):
 597             return []
 598         else:
 599             return ''