plugins/video/video.py

   1 import calendar
   2 import cgi
   3 import logging
   4 import os
   5 import re
   6 import struct
   7 import thread
   8 import time
   9 import traceback
  10 import urllib
  11 import zlib
  12 from UserDict import DictMixin
  13 from datetime import datetime, timedelta
  14 from xml.sax.saxutils import escape
  15
  16 from Cheetah.Template import Template
  17 from lrucache import LRUCache
  18
  19 import config
  20 import metadata
  21 import mind
  22 import qtfaststart
  23 import transcode
  24 from plugin import EncodeUnicode, Plugin, quote
  25
  26 logger = logging.getLogger('pyTivo.video.video')
  27
  28 SCRIPTDIR = os.path.dirname(__file__)
  29
  30 CLASS_NAME = 'Video'
  31
  32 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  33
  34 # Preload the templates
  35 def tmpl(name):
  36     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  37
  38 HTML_CONTAINER_TEMPLATE_MOBILE = tmpl('container_mob.tmpl')
  39 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  40 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  41 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  42
  43 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  44 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  45 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  46 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  47 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  48 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  49 .wm .wmd .wtv .yuv""".split()
  50
  51 use_extensions = True
  52 try:
  53     assert(config.get_bin('ffmpeg'))
  54 except:
  55     use_extensions = False
  56
  57 queue = []  # Recordings to push
  58
  59 def uniso(iso):
  60     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  61
  62 def isodt(iso):
  63     return datetime(*uniso(iso)[:6])
  64
  65 def isogm(iso):
  66     return int(calendar.timegm(uniso(iso)))
  67
  68 class Pushable(object):
  69
  70     def push_one_file(self, f):
  71         file_info = VideoDetails()
  72         file_info['valid'] = transcode.supported_format(f['path'])
  73
  74         temp_share = config.get_server('temp_share', '')
  75         temp_share_path = ''
  76         if temp_share:
  77             for name, data in config.getShares():
  78                 if temp_share == name:
  79                     temp_share_path = data.get('path')
  80
  81         mime = 'video/mpeg'
  82         if config.isHDtivo(f['tsn']):
  83             for m in ['video/mp4', 'video/bif']:
  84                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  85                     mime = m
  86                     break
  87
  88             if (mime == 'video/mpeg' and
  89                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  90                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'], temp_share_path)
  91                 if new_path:
  92                     mime = 'video/mp4'
  93                     f['name'] = new_path
  94                     if temp_share_path:
  95                         ip = config.get_ip()
  96                         port = config.getPort()
  97                         container = quote(temp_share) + '/'
  98                         f['url'] = 'http://%s:%s/%s' % (ip, port, container)
  99
 100         if file_info['valid']:
 101             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
 102
 103         url = f['url'] + quote(f['name'])
 104
 105         title = file_info['seriesTitle']
 106         if not title:
 107             title = file_info['title']
 108
 109         source = file_info['seriesId']
 110         if not source:
 111             source = title
 112
 113         subtitle = file_info['episodeTitle']
 114         try:
 115             m = mind.getMind(f['tsn'])
 116             m.pushVideo(
 117                 tsn = f['tsn'],
 118                 url = url,
 119                 description = file_info['description'],
 120                 duration = file_info['duration'] / 1000,
 121                 size = file_info['size'],
 122                 title = title,
 123                 subtitle = subtitle,
 124                 source = source,
 125                 mime = mime,
 126                 tvrating = file_info['tvRating'])
 127         except Exception, msg:
 128             logger.error(msg)
 129
 130     def process_queue(self):
 131         while queue:
 132             time.sleep(5)
 133             item = queue.pop(0)
 134             self.push_one_file(item)
 135
 136     def readip(self):
 137         """ returns your external IP address by querying dyndns.org """
 138         f = urllib.urlopen('http://checkip.dyndns.org/')
 139         s = f.read()
 140         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 141         return m.group(0)
 142
 143     def Push(self, handler, query):
 144         tsn = query['tsn'][0]
 145         for key in config.tivo_names:
 146             if config.tivo_names[key] == tsn:
 147                 tsn = key
 148                 break
 149         tivo_name = config.tivo_names.get(tsn, tsn)
 150
 151         container = quote(query['Container'][0].split('/')[0])
 152         ip = config.get_ip(tsn)
 153         port = config.getPort()
 154
 155         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 156         if config.getIsExternal(tsn):
 157             exturl = config.get_server('externalurl')
 158             if exturl:
 159                 if not exturl.endswith('/'):
 160                     exturl += '/'
 161                 baseurl = exturl + container
 162             else:
 163                 ip = self.readip()
 164                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 165
 166         path = self.get_local_base_path(handler, query)
 167
 168         files = query.get('File', [])
 169         for f in files:
 170             file_path = path + os.path.normpath(f)
 171             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 172                           'url': baseurl})
 173             if len(queue) == 1:
 174                 thread.start_new_thread(Video.process_queue, (self,))
 175
 176             logger.info('[%s] Queued "%s" for Push to %s' %
 177                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 178                          unicode(file_path, 'utf-8'), tivo_name))
 179
 180         files = [unicode(f, 'utf-8') for f in files]
 181         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 182
 183 class BaseVideo(Plugin):
 184
 185     CONTENT_TYPE = 'x-container/tivo-videos'
 186
 187     tvbus_cache = LRUCache(1)
 188
 189     def pre_cache(self, full_path):
 190         if Video.video_file_filter(self, full_path):
 191             transcode.supported_format(full_path)
 192
 193     def video_file_filter(self, full_path, type=None):
 194         if os.path.isdir(unicode(full_path, 'utf-8')):
 195             return True
 196         if use_extensions:
 197             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 198         else:
 199             return transcode.supported_format(full_path)
 200
 201     def send_file(self, handler, path, query):
 202         mime = 'video/x-tivo-mpeg'
 203         tsn = handler.headers.getheader('tsn', '')
 204         tivo_name = config.tivo_names.get(tsn, tsn)
 205
 206         is_tivo_file = (path[-5:].lower() == '.tivo')
 207
 208         if 'Format' in query:
 209             mime = query['Format'][0]
 210
 211         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 212         compatible = (not needs_tivodecode and
 213                       transcode.tivo_compatible(path, tsn, mime)[0])
 214
 215         try:  # "bytes=XXX-"
 216             offset = int(handler.headers.getheader('Range')[6:-1])
 217         except:
 218             offset = 0
 219
 220         if needs_tivodecode:
 221             valid = bool(config.get_bin('tivodecode') and
 222                          config.get_server('tivo_mak'))
 223         else:
 224             valid = True
 225
 226         if valid and offset:
 227             valid = ((compatible and offset < os.stat(path).st_size) or
 228                      (not compatible and transcode.is_resumable(path, offset)))
 229
 230         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 231         faking = (mime == 'video/x-tivo-mpeg' and
 232                   not (is_tivo_file and compatible))
 233         fname = unicode(path, 'utf-8')
 234         thead = ''
 235         if faking:
 236             thead = self.tivo_header(tsn, path, mime)
 237         if compatible:
 238             size = os.stat(fname).st_size + len(thead)
 239             handler.send_response(200)
 240             handler.send_header('Content-Length', size - offset)
 241             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 242                                 (offset, size - offset - 1, size))
 243         else:
 244             handler.send_response(206)
 245             handler.send_header('Transfer-Encoding', 'chunked')
 246         handler.send_header('Content-Type', mime)
 247         handler.send_header('Connection', 'close')
 248         handler.end_headers()
 249
 250         logger.info('[%s] Start sending "%s" to %s' %
 251                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 252         start = time.time()
 253         count = 0
 254
 255         if valid:
 256             if compatible:
 257                 if faking and not offset:
 258                     handler.wfile.write(thead)
 259                 logger.debug('"%s" is tivo compatible' % fname)
 260                 f = open(fname, 'rb')
 261                 try:
 262                     if mime == 'video/mp4':
 263                         count = qtfaststart.process(f, handler.wfile, offset)
 264                     else:
 265                         if offset:
 266                             offset -= len(thead)
 267                             f.seek(offset)
 268                         while True:
 269                             block = f.read(512 * 1024)
 270                             if not block:
 271                                 break
 272                             handler.wfile.write(block)
 273                             count += len(block)
 274                 except Exception, msg:
 275                     logger.info(msg)
 276                 f.close()
 277             else:
 278                 logger.debug('"%s" is not tivo compatible' % fname)
 279                 if offset:
 280                     count = transcode.resume_transfer(path, handler.wfile,
 281                                                       offset)
 282                 else:
 283                     count = transcode.transcode(False, path, handler.wfile,
 284                                                 tsn, mime, thead)
 285         try:
 286             if not compatible:
 287                  handler.wfile.write('0\r\n\r\n')
 288             handler.wfile.flush()
 289         except Exception, msg:
 290             logger.info(msg)
 291
 292         mega_elapsed = (time.time() - start) * 1024 * 1024
 293         if mega_elapsed < 1:
 294             mega_elapsed = 1
 295         rate = count * 8.0 / mega_elapsed
 296         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 297                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 298                      tivo_name, count, rate))
 299
 300         if fname.endswith('.pyTivo-temp'):
 301             os.remove(fname)
 302
 303     def __duration(self, full_path):
 304         return transcode.video_info(full_path)['millisecs']
 305
 306     def __total_items(self, full_path):
 307         count = 0
 308         try:
 309             full_path = unicode(full_path, 'utf-8')
 310             for f in os.listdir(full_path):
 311                 if f.startswith('.'):
 312                     continue
 313                 f = os.path.join(full_path, f)
 314                 f2 = f.encode('utf-8')
 315                 if os.path.isdir(f):
 316                     count += 1
 317                 elif use_extensions:
 318                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 319                         count += 1
 320                 elif f2 in transcode.info_cache:
 321                     if transcode.supported_format(f2):
 322                         count += 1
 323         except:
 324             pass
 325         return count
 326
 327     def __est_size(self, full_path, tsn='', mime=''):
 328         # Size is estimated by taking audio and video bit rate adding 2%
 329
 330         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 331             return int(os.stat(unicode(full_path, 'utf-8')).st_size)
 332         else:
 333             # Must be re-encoded
 334             if config.get_tsn('audio_codec', tsn) == None:
 335                 audioBPS = config.getMaxAudioBR(tsn) * 1000
 336             else:
 337                 audioBPS = config.strtod(config.getAudioBR(tsn))
 338             videoBPS = transcode.select_videostr(full_path, tsn)
 339             bitrate =  audioBPS + videoBPS
 340             return int((self.__duration(full_path) / 1000) *
 341                        (bitrate * 1.02 / 8))
 342
 343     def metadata_full(self, full_path, tsn='', mime=''):
 344         data = {}
 345         vInfo = transcode.video_info(full_path)
 346
 347         if ((int(vInfo['vHeight']) >= 720 and
 348              config.getTivoHeight >= 720) or
 349             (int(vInfo['vWidth']) >= 1280 and
 350              config.getTivoWidth >= 1280)):
 351             data['showingBits'] = '4096'
 352
 353         data.update(metadata.basic(full_path))
 354         if full_path[-5:].lower() == '.tivo':
 355             data.update(metadata.from_tivo(full_path))
 356         if full_path[-4:].lower() == '.wtv':
 357             data.update(metadata.from_mscore(vInfo['rawmeta']))
 358
 359         if 'episodeNumber' in data:
 360             try:
 361                 ep = int(data['episodeNumber'])
 362             except:
 363                 ep = 0
 364             data['episodeNumber'] = str(ep)
 365
 366         if config.getDebug() and 'vHost' not in data:
 367             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 368             if compatible:
 369                 transcode_options = {}
 370             else:
 371                 transcode_options = transcode.transcode(True, full_path,
 372                                                         '', tsn, mime)
 373             data['vHost'] = (
 374                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 375                 ['SOURCE INFO: '] +
 376                 ["%s=%s" % (k, v)
 377                  for k, v in sorted(vInfo.items(), reverse=True)] +
 378                 ['TRANSCODE OPTIONS: '] +
 379                 ["%s" % (v) for k, v in transcode_options.items()] +
 380                 ['SOURCE FILE: ', os.path.basename(full_path)]
 381             )
 382
 383         now = datetime.utcnow()
 384         if 'time' in data:
 385             if data['time'].lower() == 'file':
 386                 mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 387                 if (mtime < 0):
 388                     mtime = 0
 389                 try:
 390                     now = datetime.utcfromtimestamp(mtime)
 391                 except:
 392                     logger.warning('Bad file time on ' + full_path)
 393             elif data['time'].lower() == 'oad':
 394                     now = isodt(data['originalAirDate'])
 395             else:
 396                 try:
 397                     now = isodt(data['time'])
 398                 except:
 399                     logger.warning('Bad time format: ' + data['time'] +
 400                                    ' , using current time')
 401
 402         duration = self.__duration(full_path)
 403         duration_delta = timedelta(milliseconds = duration)
 404         min = duration_delta.seconds / 60
 405         sec = duration_delta.seconds % 60
 406         hours = min / 60
 407         min = min % 60
 408
 409         data.update({'time': now.isoformat(),
 410                      'startTime': now.isoformat(),
 411                      'stopTime': (now + duration_delta).isoformat(),
 412                      'size': self.__est_size(full_path, tsn, mime),
 413                      'duration': duration,
 414                      'iso_duration': ('P%sDT%sH%sM%sS' %
 415                           (duration_delta.days, hours, min, sec))})
 416
 417         return data
 418
 419     def QueryContainer(self, handler, query):
 420         tsn = handler.headers.getheader('tsn', '')
 421         subcname = query['Container'][0]
 422         useragent = handler.headers.getheader('User-Agent', '')
 423
 424         if not self.get_local_path(handler, query):
 425             handler.send_error(404)
 426             return
 427
 428         container = handler.container
 429         precache = container.get('precache', 'False').lower() == 'true'
 430         force_alpha = container.get('force_alpha', 'False').lower() == 'true'
 431         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 432
 433         files, total, start = self.get_files(handler, query,
 434                                              self.video_file_filter,
 435                                              force_alpha)
 436
 437         videos = []
 438         local_base_path = self.get_local_base_path(handler, query)
 439         for f in files:
 440             video = VideoDetails()
 441             mtime = f.mdate
 442             try:
 443                 ltime = time.localtime(mtime)
 444             except:
 445                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 446                 mtime = int(time.time())
 447                 ltime = time.localtime(mtime)
 448             video['captureDate'] = hex(mtime)
 449             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 450             video['name'] = os.path.basename(f.name)
 451             video['path'] = f.name
 452             video['part_path'] = f.name.replace(local_base_path, '', 1)
 453             if not video['part_path'].startswith(os.path.sep):
 454                 video['part_path'] = os.path.sep + video['part_path']
 455             video['title'] = os.path.basename(f.name)
 456             video['is_dir'] = f.isdir
 457             if video['is_dir']:
 458                 video['small_path'] = subcname + '/' + video['name']
 459                 video['total_items'] = self.__total_items(f.name)
 460             else:
 461                 if precache or len(files) == 1 or f.name in transcode.info_cache:
 462                     video['valid'] = transcode.supported_format(f.name)
 463                     if video['valid']:
 464                         video.update(self.metadata_full(f.name, tsn))
 465                         if len(files) == 1:
 466                             video['captureDate'] = hex(isogm(video['time']))
 467                 else:
 468                     video['valid'] = True
 469                     video.update(metadata.basic(f.name))
 470
 471                 if self.use_ts(tsn, f.name):
 472                     video['mime'] = 'video/x-tivo-mpeg-ts'
 473                 else:
 474                     video['mime'] = 'video/x-tivo-mpeg'
 475
 476                 video['textSize'] = ( '%.3f GB' %
 477                     (float(f.size) / (1024 ** 3)) )
 478
 479             videos.append(video)
 480
 481         logger.debug('mobileagent: %d useragent: %s' % (useragent.lower().find('mobile'), useragent.lower()))
 482         use_mobile = useragent.lower().find('mobile') > 0
 483         if use_html:
 484             if use_mobile:
 485                 t = Template(HTML_CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 486             else:
 487                 t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 488         else:
 489             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 490
 491         t.container = handler.cname
 492         t.name = subcname
 493         t.total = total
 494         t.start = start
 495         t.videos = videos
 496         t.quote = quote
 497         t.escape = escape
 498         t.crc = zlib.crc32
 499         t.guid = config.getGUID()
 500         t.tivos = config.tivos
 501         t.tivo_names = config.tivo_names
 502         if use_html:
 503             handler.send_html(str(t))
 504         else:
 505             handler.send_xml(str(t))
 506
 507     def use_ts(self, tsn, file_path):
 508         if config.is_ts_capable(tsn):
 509             if file_path[-5:].lower() == '.tivo':
 510                 try:
 511                     flag = file(file_path).read(8)
 512                 except:
 513                     return False
 514                 if ord(flag[7]) & 0x20:
 515                     return True
 516             elif config.has_ts_flag():
 517                 return True
 518
 519         return False
 520
 521     def get_details_xml(self, tsn, file_path):
 522         if (tsn, file_path) in self.tvbus_cache:
 523             details = self.tvbus_cache[(tsn, file_path)]
 524         else:
 525             file_info = VideoDetails()
 526             file_info['valid'] = transcode.supported_format(file_path)
 527             if file_info['valid']:
 528                 file_info.update(self.metadata_full(file_path, tsn))
 529
 530             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 531             t.video = file_info
 532             t.escape = escape
 533             t.get_tv = metadata.get_tv
 534             t.get_mpaa = metadata.get_mpaa
 535             t.get_stars = metadata.get_stars
 536             details = str(t)
 537             self.tvbus_cache[(tsn, file_path)] = details
 538         return details
 539
 540     def tivo_header(self, tsn, path, mime):
 541         if mime == 'video/x-tivo-mpeg-ts':
 542             flag = 45
 543         else:
 544             flag = 13
 545         details = self.get_details_xml(tsn, path)
 546         ld = len(details)
 547         chunklen = ld * 2 + 44
 548         padding = 2048 - chunklen % 1024
 549
 550         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 551                                             padding + chunklen, 2),
 552                         struct.pack('>LLHH', ld + 16, ld, 1, 0),
 553                         details, '\0' * 4,
 554                         struct.pack('>LLHH', ld + 19, ld, 2, 0),
 555                         details, '\0' * padding])
 556
 557     def TVBusQuery(self, handler, query):
 558         tsn = handler.headers.getheader('tsn', '')
 559         f = query['File'][0]
 560         path = self.get_local_path(handler, query)
 561         file_path = path + os.path.normpath(f)
 562
 563         details = self.get_details_xml(tsn, file_path)
 564
 565         handler.send_xml(details)
 566
 567 class Video(BaseVideo, Pushable):
 568         pass
 569
 570 class VideoDetails(DictMixin):
 571
 572     def __init__(self, d=None):
 573         if d:
 574             self.d = d
 575         else:
 576             self.d = {}
 577
 578     def __getitem__(self, key):
 579         if key not in self.d:
 580             self.d[key] = self.default(key)
 581         return self.d[key]
 582
 583     def __contains__(self, key):
 584         return True
 585
 586     def __setitem__(self, key, value):
 587         self.d[key] = value
 588
 589     def __delitem__(self):
 590         del self.d[key]
 591
 592     def keys(self):
 593         return self.d.keys()
 594
 595     def __iter__(self):
 596         return self.d.__iter__()
 597
 598     def iteritems(self):
 599         return self.d.iteritems()
 600
 601     def default(self, key):
 602         defaults = {
 603             'showingBits' : '0',
 604             'displayMajorNumber' : '0',
 605             'displayMinorNumber' : '0',
 606             'isEpisode' : 'true',
 607             'colorCode' : ('COLOR', '4'),
 608             'showType' : ('SERIES', '5')
 609         }
 610         if key in defaults:
 611             return defaults[key]
 612         elif key.startswith('v'):
 613             return []
 614         else:
 615             return ''