plugins/video/video.py

   1 import cgi
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import traceback
   9 import urllib
  10 import zlib
  11 from UserDict import DictMixin
  12 from datetime import datetime, timedelta
  13 from xml.sax.saxutils import escape
  14
  15 from Cheetah.Template import Template
  16 from lrucache import LRUCache
  17
  18 import config
  19 import metadata
  20 import mind
  21 import qtfaststart
  22 import transcode
  23 from plugin import EncodeUnicode, Plugin, quote
  24
  25 logger = logging.getLogger('pyTivo.video.video')
  26
  27 SCRIPTDIR = os.path.dirname(__file__)
  28
  29 CLASS_NAME = 'Video'
  30
  31 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  32
  33 # Preload the templates
  34 def tmpl(name):
  35     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  36
  37 HTML_CONTAINER_TEMPLATE_MOBILE = tmpl('container_mob.tmpl')
  38 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  39 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  40 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  41
  42 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .vob .mp4 .m4v .mkv .ts
  43 .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf .dat
  44 .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf .m1v
  45 .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21 .mpe
  46 .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb .rts
  47 .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm .wm
  48 .wmd .yuv""".split()
  49
  50 use_extensions = True
  51 try:
  52     assert(config.get_bin('ffmpeg'))
  53 except:
  54     use_extensions = False
  55
  56 queue = []  # Recordings to push
  57
  58 class Video(Plugin):
  59
  60     CONTENT_TYPE = 'x-container/tivo-videos'
  61
  62     tvbus_cache = LRUCache(1)
  63
  64     def pre_cache(self, full_path):
  65         if Video.video_file_filter(self, full_path):
  66             transcode.supported_format(full_path)
  67
  68     def video_file_filter(self, full_path, type=None):
  69         if os.path.isdir(unicode(full_path, 'utf-8')):
  70             return True
  71         if use_extensions:
  72             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
  73         else:
  74             return transcode.supported_format(full_path)
  75
  76     def send_file(self, handler, path, query):
  77         mime = 'video/mpeg'
  78         tsn = handler.headers.getheader('tsn', '')
  79         tivo_name = config.tivo_names.get(tsn, tsn)
  80
  81         is_tivo_file = (path[-5:].lower() == '.tivo')
  82
  83         if is_tivo_file and transcode.tivo_compatible(path, tsn, mime)[0]:
  84             mime = 'video/x-tivo-mpeg'
  85
  86         if 'Format' in query:
  87             mime = query['Format'][0]
  88
  89         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
  90         compatible = (not needs_tivodecode and
  91                       transcode.tivo_compatible(path, tsn, mime)[0])
  92
  93         offset = handler.headers.getheader('Range')
  94         if offset:
  95             offset = int(offset[6:-1])  # "bytes=XXX-"
  96
  97         if needs_tivodecode:
  98             valid = bool(config.get_bin('tivodecode') and
  99                          config.get_server('tivo_mak'))
 100         else:
 101             valid = True
 102
 103         if valid and offset:
 104             valid = ((compatible and offset < os.stat(path).st_size) or
 105                      (not compatible and transcode.is_resumable(path, offset)))
 106
 107         fname = unicode(path, 'utf-8')
 108         handler.send_response(206)
 109         handler.send_header('Content-Type', mime)
 110         handler.send_header('Connection', 'close')
 111         if compatible:
 112             handler.send_header('Content-Length',
 113                                 os.stat(fname).st_size - offset)
 114         else:
 115             handler.send_header('Transfer-Encoding', 'chunked')
 116         handler.end_headers()
 117
 118         logger.info('[%s] Start sending "%s" to %s' %
 119                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 120         start = time.time()
 121         count = 0
 122
 123         if valid:
 124             if compatible:
 125                 logger.debug('"%s" is tivo compatible' % fname)
 126                 f = open(fname, 'rb')
 127                 try:
 128                     if mime == 'video/mp4':
 129                         count = qtfaststart.process(f, handler.wfile, offset)
 130                     else:
 131                         if offset:
 132                             f.seek(offset)
 133                         while True:
 134                             block = f.read(512 * 1024)
 135                             if not block:
 136                                 break
 137                             handler.wfile.write(block)
 138                             count += len(block)
 139                 except Exception, msg:
 140                     logger.info(msg)
 141                 f.close()
 142             else:
 143                 logger.debug('"%s" is not tivo compatible' % fname)
 144                 if offset:
 145                     count = transcode.resume_transfer(path, handler.wfile,
 146                                                       offset)
 147                 else:
 148                     count = transcode.transcode(False, path,
 149                                                 handler.wfile, tsn)
 150         try:
 151             if not compatible:
 152                  handler.wfile.write('0\r\n\r\n')
 153             handler.wfile.flush()
 154         except Exception, msg:
 155             logger.info(msg)
 156
 157         mega_elapsed = (time.time() - start) * 1024 * 1024
 158         if mega_elapsed < 1:
 159             mega_elapsed = 1
 160         rate = count * 8.0 / mega_elapsed
 161         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 162                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 163                      tivo_name, count, rate))
 164
 165         if fname.endswith('.pyTivo-temp'):
 166             os.remove(fname)
 167
 168     def __duration(self, full_path):
 169         return transcode.video_info(full_path)['millisecs']
 170
 171     def __total_items(self, full_path):
 172         count = 0
 173         try:
 174             full_path = unicode(full_path, 'utf-8')
 175             for f in os.listdir(full_path):
 176                 if f.startswith('.'):
 177                     continue
 178                 f = os.path.join(full_path, f)
 179                 f2 = f.encode('utf-8')
 180                 if os.path.isdir(f):
 181                     count += 1
 182                 elif use_extensions:
 183                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 184                         count += 1
 185                 elif f2 in transcode.info_cache:
 186                     if transcode.supported_format(f2):
 187                         count += 1
 188         except:
 189             pass
 190         return count
 191
 192     def __est_size(self, full_path, tsn='', mime=''):
 193         # Size is estimated by taking audio and video bit rate adding 2%
 194
 195         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 196             return int(os.stat(unicode(full_path, 'utf-8')).st_size)
 197         else:
 198             # Must be re-encoded
 199             if config.get_tsn('audio_codec', tsn) == None:
 200                 audioBPS = config.getMaxAudioBR(tsn) * 1000
 201             else:
 202                 audioBPS = config.strtod(config.getAudioBR(tsn))
 203             videoBPS = transcode.select_videostr(full_path, tsn)
 204             bitrate =  audioBPS + videoBPS
 205             return int((self.__duration(full_path) / 1000) *
 206                        (bitrate * 1.02 / 8))
 207
 208     def metadata_full(self, full_path, tsn='', mime=''):
 209         data = {}
 210         vInfo = transcode.video_info(full_path)
 211
 212         if ((int(vInfo['vHeight']) >= 720 and
 213              config.getTivoHeight >= 720) or
 214             (int(vInfo['vWidth']) >= 1280 and
 215              config.getTivoWidth >= 1280)):
 216             data['showingBits'] = '4096'
 217
 218         data.update(metadata.basic(full_path))
 219         if full_path[-5:].lower() == '.tivo':
 220             data.update(metadata.from_tivo(full_path))
 221
 222         if 'episodeNumber' in data:
 223             try:
 224                 ep = int(data['episodeNumber'])
 225             except:
 226                 ep = 0
 227             data['episodeNumber'] = str(ep)
 228
 229         if config.getDebug() and 'vHost' not in data:
 230             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 231             if compatible:
 232                 transcode_options = {}
 233             else:
 234                 transcode_options = transcode.transcode(True, full_path,
 235                                                         '', tsn)
 236             data['vHost'] = (
 237                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 238                 ['SOURCE INFO: '] +
 239                 ["%s=%s" % (k, v)
 240                  for k, v in sorted(vInfo.items(), reverse=True)] +
 241                 ['TRANSCODE OPTIONS: '] +
 242                 ["%s" % (v) for k, v in transcode_options.items()] +
 243                 ['SOURCE FILE: ', os.path.split(full_path)[1]]
 244             )
 245
 246         now = datetime.utcnow()
 247         if 'time' in data:
 248             if data['time'].lower() == 'file':
 249                 mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 250                 if (mtime < 0):
 251                     mtime = 0
 252                 try:
 253                     now = datetime.utcfromtimestamp(mtime)
 254                 except:
 255                     logger.warning('Bad file time on ' + full_path)
 256             elif data['time'].lower() == 'oad':
 257                     now = datetime.strptime(data['originalAirDate'][:19],
 258                                             '%Y-%m-%dT%H:%M:%S')
 259             else:
 260                 try:
 261                     now = datetime.strptime(data['time'][:19],
 262                                             '%Y-%m-%dT%H:%M:%S')
 263                 except:
 264                     logger.warning('Bad time format: ' + data['time'] +
 265                                    ' , using current time')
 266
 267         duration = self.__duration(full_path)
 268         duration_delta = timedelta(milliseconds = duration)
 269         min = duration_delta.seconds / 60
 270         sec = duration_delta.seconds % 60
 271         hours = min / 60
 272         min = min % 60
 273
 274         data.update({'time': now.isoformat(),
 275                      'startTime': now.isoformat(),
 276                      'stopTime': (now + duration_delta).isoformat(),
 277                      'size': self.__est_size(full_path, tsn, mime),
 278                      'duration': duration,
 279                      'iso_duration': ('P%sDT%sH%sM%sS' %
 280                           (duration_delta.days, hours, min, sec))})
 281
 282         return data
 283
 284     def QueryContainer(self, handler, query):
 285         tsn = handler.headers.getheader('tsn', '')
 286         subcname = query['Container'][0]
 287         cname = subcname.split('/')[0]
 288         useragent = handler.headers.getheader('User-Agent', '')
 289
 290         if (not cname in handler.server.containers or
 291             not self.get_local_path(handler, query)):
 292             handler.send_error(404)
 293             return
 294
 295         container = handler.server.containers[cname]
 296         precache = container.get('precache', 'False').lower() == 'true'
 297         force_alpha = container.get('force_alpha', 'False').lower() == 'true'
 298
 299         files, total, start = self.get_files(handler, query,
 300                                              self.video_file_filter,
 301                                              force_alpha)
 302
 303         videos = []
 304         local_base_path = self.get_local_base_path(handler, query)
 305         for f in files:
 306             video = VideoDetails()
 307             mtime = f.mdate
 308             try:
 309                 ltime = time.localtime(mtime)
 310             except:
 311                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 312                 mtime = int(time.time())
 313                 ltime = time.localtime(mtime)
 314             video['captureDate'] = hex(mtime)
 315             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 316             video['name'] = os.path.split(f.name)[1]
 317             video['path'] = f.name
 318             video['part_path'] = f.name.replace(local_base_path, '', 1)
 319             if not video['part_path'].startswith(os.path.sep):
 320                 video['part_path'] = os.path.sep + video['part_path']
 321             video['title'] = os.path.split(f.name)[1]
 322             video['is_dir'] = f.isdir
 323             if video['is_dir']:
 324                 video['small_path'] = subcname + '/' + video['name']
 325                 video['total_items'] = self.__total_items(f.name)
 326             else:
 327                 if precache or len(files) == 1 or f.name in transcode.info_cache:
 328                     video['valid'] = transcode.supported_format(f.name)
 329                     if video['valid']:
 330                         video.update(self.metadata_full(f.name, tsn))
 331                 else:
 332                     video['valid'] = True
 333                     video.update(metadata.basic(f.name))
 334
 335                 video['textSize'] = ( '%.3f GB' %
 336                     (float(f.size) / (1024 ** 3)) )
 337
 338             videos.append(video)
 339
 340         logger.debug('mobileagent: %d useragent: %s' % (useragent.lower().find('mobile'), useragent.lower()))
 341         if tsn:
 342             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 343         elif useragent.lower().find('mobile') > 0:
 344             t = Template(HTML_CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 345         else:
 346             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 347         t.container = cname
 348         t.name = subcname
 349         t.total = total
 350         t.start = start
 351         t.videos = videos
 352         t.quote = quote
 353         t.escape = escape
 354         t.crc = zlib.crc32
 355         t.guid = config.getGUID()
 356         t.tivos = config.tivos
 357         t.tivo_names = config.tivo_names
 358         handler.send_response(200)
 359         if tsn:
 360             handler.send_header('Content-Type', 'text/xml')
 361         else:
 362             handler.send_header('Content-Type', 'text/html; charset=utf-8')
 363         handler.send_header('Expires', '0')
 364         handler.end_headers()
 365         handler.wfile.write(t)
 366
 367     def get_details_xml(self, tsn, file_path):
 368         if (tsn, file_path) in self.tvbus_cache:
 369             details = self.tvbus_cache[(tsn, file_path)]
 370         else:
 371             file_info = VideoDetails()
 372             file_info['valid'] = transcode.supported_format(file_path)
 373             if file_info['valid']:
 374                 file_info.update(self.metadata_full(file_path, tsn))
 375
 376             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 377             t.video = file_info
 378             t.escape = escape
 379             details = str(t)
 380             self.tvbus_cache[(tsn, file_path)] = details
 381         return details
 382
 383     def tivo_header(self, tsn, path, flag=13):
 384         details = self.get_details_xml(tsn, path)
 385         ld = len(details)
 386         chunklen = ld * 2 + 44
 387         padding = 2048 - chunklen % 1024
 388
 389         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 390                                             padding + chunklen, 2),
 391                         struct.pack('>LLHH', ld + 16, ld, 1, 0),
 392                         details, '\0' * 4,
 393                         struct.pack('>LLHH', ld + 19, ld, 2, 0),
 394                         details, '\0' * padding])
 395
 396     def TVBusQuery(self, handler, query):
 397         tsn = handler.headers.getheader('tsn', '')
 398         f = query['File'][0]
 399         path = self.get_local_path(handler, query)
 400         file_path = path + os.path.normpath(f)
 401
 402         details = self.get_details_xml(tsn, file_path)
 403
 404         handler.send_response(200)
 405         handler.send_header('Content-Type', 'text/xml')
 406         handler.send_header('Content-Length', len(details))
 407         handler.send_header('Connection', 'close')
 408         handler.send_header('Expires', '0')
 409         handler.end_headers()
 410         handler.wfile.write(details)
 411
 412     def push_one_file(self, f):
 413         file_info = VideoDetails()
 414         file_info['valid'] = transcode.supported_format(f['path'])
 415
 416         temp_share = config.get_server('temp_share', '')
 417         temp_share_path = ''
 418         if temp_share:
 419             for name, data in config.getShares():
 420                 if temp_share == name:
 421                     temp_share_path = data.get('path')
 422
 423         mime = 'video/mpeg'
 424         if config.isHDtivo(f['tsn']):
 425             for m in ['video/mp4', 'video/bif']:
 426                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
 427                     mime = m
 428                     break
 429
 430             if (mime == 'video/mpeg' and
 431                 transcode.mp4_remuxable(f['path'], f['tsn'])):
 432                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'], temp_share_path)
 433                 if new_path:
 434                     mime = 'video/mp4'
 435                     f['name'] = new_path
 436                     if temp_share_path:
 437                         ip = config.get_ip()
 438                         port = config.getPort()
 439                         container = quote(temp_share) + '/'
 440                         f['url'] = 'http://%s:%s/%s' % (ip, port, container)
 441
 442         if file_info['valid']:
 443             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
 444
 445         url = f['url'] + quote(f['name'])
 446
 447         title = file_info['seriesTitle']
 448         if not title:
 449             title = file_info['title']
 450
 451         source = file_info['seriesId']
 452         if not source:
 453             source = title
 454
 455         subtitle = file_info['episodeTitle']
 456         try:
 457             m = mind.getMind(f['tsn'])
 458             m.pushVideo(
 459                 tsn = f['tsn'],
 460                 url = url,
 461                 description = file_info['description'],
 462                 duration = file_info['duration'] / 1000,
 463                 size = file_info['size'],
 464                 title = title,
 465                 subtitle = subtitle,
 466                 source = source,
 467                 mime = mime,
 468                 tvrating = file_info['tvRating'])
 469         except Exception, msg:
 470             logger.error(msg)
 471
 472     def process_queue(self):
 473         while queue:
 474             time.sleep(5)
 475             item = queue.pop(0)
 476             self.push_one_file(item)
 477
 478     def Push(self, handler, query):
 479         tsn = query['tsn'][0]
 480         for key in config.tivo_names:
 481             if config.tivo_names[key] == tsn:
 482                 tsn = key
 483                 break
 484         tivo_name = config.tivo_names.get(tsn, tsn)
 485
 486         container = quote(query['Container'][0].split('/')[0])
 487         ip = config.get_ip()
 488         port = config.getPort()
 489
 490         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 491         if config.getIsExternal(tsn):
 492             exturl = config.get_server('externalurl')
 493             if exturl:
 494                 baseurl = exturl
 495             else:
 496                 ip = self.readip()
 497                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 498
 499         path = self.get_local_base_path(handler, query)
 500
 501         files = query.get('File', [])
 502         for f in files:
 503             file_path = path + os.path.normpath(f)
 504             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 505                           'url': baseurl})
 506             if len(queue) == 1:
 507                 thread.start_new_thread(Video.process_queue, (self,))
 508
 509             logger.info('[%s] Queued "%s" for Push to %s' %
 510                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 511                          unicode(file_path, 'utf-8'), tivo_name))
 512
 513         files = [unicode(f, 'utf-8') for f in files]
 514         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 515
 516     def readip(self):
 517         """ returns your external IP address by querying dyndns.org """
 518         f = urllib.urlopen('http://checkip.dyndns.org/')
 519         s = f.read()
 520         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 521         return m.group(0)
 522
 523 class VideoDetails(DictMixin):
 524
 525     def __init__(self, d=None):
 526         if d:
 527             self.d = d
 528         else:
 529             self.d = {}
 530
 531     def __getitem__(self, key):
 532         if key not in self.d:
 533             self.d[key] = self.default(key)
 534         return self.d[key]
 535
 536     def __contains__(self, key):
 537         return True
 538
 539     def __setitem__(self, key, value):
 540         self.d[key] = value
 541
 542     def __delitem__(self):
 543         del self.d[key]
 544
 545     def keys(self):
 546         return self.d.keys()
 547
 548     def __iter__(self):
 549         return self.d.__iter__()
 550
 551     def iteritems(self):
 552         return self.d.iteritems()
 553
 554     def default(self, key):
 555         defaults = {
 556             'showingBits' : '0',
 557             'episodeNumber' : '0',
 558             'displayMajorNumber' : '0',
 559             'displayMinorNumber' : '0',
 560             'isEpisode' : 'true',
 561             'colorCode' : ('COLOR', '4'),
 562             'showType' : ('SERIES', '5')
 563         }
 564         if key in defaults:
 565             return defaults[key]
 566         elif key.startswith('v'):
 567             return []
 568         else:
 569             return ''