plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 logger = logging.getLogger('pyTivo.video.video')
  25
  26 SCRIPTDIR = os.path.dirname(__file__)
  27
  28 CLASS_NAME = 'Video'
  29
  30 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  31
  32 # Preload the templates
  33 def tmpl(name):
  34     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  35
  36 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  37 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  38 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  39
  40 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  41 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  42 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  43 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  44 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  45 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  46 .wm .wmd .wtv .yuv""".split()
  47
  48 use_extensions = True
  49 try:
  50     assert(config.get_bin('ffmpeg'))
  51 except:
  52     use_extensions = False
  53
  54 queue = []  # Recordings to push
  55
  56 def uniso(iso):
  57     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  58
  59 def isodt(iso):
  60     return datetime(*uniso(iso)[:6])
  61
  62 def isogm(iso):
  63     return int(calendar.timegm(uniso(iso)))
  64
  65 class Pushable(object):
  66
  67     def push_one_file(self, f):
  68         file_info = VideoDetails()
  69         file_info['valid'] = transcode.supported_format(f['path'])
  70
  71         mime = 'video/mpeg'
  72         if config.isHDtivo(f['tsn']):
  73             for m in ['video/mp4', 'video/bif']:
  74                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  75                     mime = m
  76                     break
  77
  78             if (mime == 'video/mpeg' and
  79                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  80                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'])
  81                 if new_path:
  82                     mime = 'video/mp4'
  83                     f['name'] = new_path
  84
  85         if file_info['valid']:
  86             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
  87
  88         url = f['url'] + quote(f['name'])
  89
  90         title = file_info['seriesTitle']
  91         if not title:
  92             title = file_info['title']
  93
  94         source = file_info['seriesId']
  95         if not source:
  96             source = title
  97
  98         subtitle = file_info['episodeTitle']
  99         try:
 100             m = mind.getMind(f['tsn'])
 101             m.pushVideo(
 102                 tsn = f['tsn'],
 103                 url = url,
 104                 description = file_info['description'],
 105                 duration = file_info['duration'] / 1000,
 106                 size = file_info['size'],
 107                 title = title,
 108                 subtitle = subtitle,
 109                 source = source,
 110                 mime = mime,
 111                 tvrating = file_info['tvRating'])
 112         except Exception, msg:
 113             logger.error(msg)
 114
 115     def process_queue(self):
 116         while queue:
 117             time.sleep(5)
 118             item = queue.pop(0)
 119             self.push_one_file(item)
 120
 121     def readip(self):
 122         """ returns your external IP address by querying dyndns.org """
 123         f = urllib.urlopen('http://checkip.dyndns.org/')
 124         s = f.read()
 125         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 126         return m.group(0)
 127
 128     def Push(self, handler, query):
 129         tsn = query['tsn'][0]
 130         for key in config.tivo_names:
 131             if config.tivo_names[key] == tsn:
 132                 tsn = key
 133                 break
 134         tivo_name = config.tivo_names.get(tsn, tsn)
 135
 136         container = quote(query['Container'][0].split('/')[0])
 137         ip = config.get_ip(tsn)
 138         port = config.getPort()
 139
 140         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 141         if config.getIsExternal(tsn):
 142             exturl = config.get_server('externalurl')
 143             if exturl:
 144                 if not exturl.endswith('/'):
 145                     exturl += '/'
 146                 baseurl = exturl + container
 147             else:
 148                 ip = self.readip()
 149                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 150
 151         path = self.get_local_base_path(handler, query)
 152
 153         files = query.get('File', [])
 154         for f in files:
 155             file_path = os.path.normpath(path + '/' + f)
 156             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 157                           'url': baseurl})
 158             if len(queue) == 1:
 159                 thread.start_new_thread(Video.process_queue, (self,))
 160
 161             logger.info('[%s] Queued "%s" for Push to %s' %
 162                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 163                          unicode(file_path, 'utf-8'), tivo_name))
 164
 165         files = [unicode(f, 'utf-8') for f in files]
 166         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 167
 168 class BaseVideo(Plugin):
 169
 170     CONTENT_TYPE = 'x-container/tivo-videos'
 171
 172     tvbus_cache = LRUCache(1)
 173
 174     def video_file_filter(self, full_path, type=None):
 175         if os.path.isdir(unicode(full_path, 'utf-8')):
 176             return True
 177         if use_extensions:
 178             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 179         else:
 180             return transcode.supported_format(full_path)
 181
 182     def send_file(self, handler, path, query):
 183         mime = 'video/x-tivo-mpeg'
 184         tsn = handler.headers.getheader('tsn', '')
 185         tivo_name = config.tivo_names.get(tsn, tsn)
 186
 187         is_tivo_file = (path[-5:].lower() == '.tivo')
 188
 189         if 'Format' in query:
 190             mime = query['Format'][0]
 191
 192         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 193         compatible = (not needs_tivodecode and
 194                       transcode.tivo_compatible(path, tsn, mime)[0])
 195
 196         try:  # "bytes=XXX-"
 197             offset = int(handler.headers.getheader('Range')[6:-1])
 198         except:
 199             offset = 0
 200
 201         if needs_tivodecode:
 202             valid = bool(config.get_bin('tivodecode') and
 203                          config.get_server('tivo_mak'))
 204         else:
 205             valid = True
 206
 207         if valid and offset:
 208             valid = ((compatible and offset < os.path.getsize(path)) or
 209                      (not compatible and transcode.is_resumable(path, offset)))
 210
 211         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 212         faking = (mime == 'video/x-tivo-mpeg' and
 213                   not (is_tivo_file and compatible))
 214         fname = unicode(path, 'utf-8')
 215         thead = ''
 216         if faking:
 217             thead = self.tivo_header(tsn, path, mime)
 218         if compatible:
 219             size = os.path.getsize(fname) + len(thead)
 220             handler.send_response(200)
 221             handler.send_header('Content-Length', size - offset)
 222             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 223                                 (offset, size - offset - 1, size))
 224         else:
 225             handler.send_response(206)
 226             handler.send_header('Transfer-Encoding', 'chunked')
 227         handler.send_header('Content-Type', mime)
 228         handler.end_headers()
 229
 230         logger.info('[%s] Start sending "%s" to %s' %
 231                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 232         start = time.time()
 233         count = 0
 234
 235         if valid:
 236             if compatible:
 237                 if faking and not offset:
 238                     handler.wfile.write(thead)
 239                 logger.debug('"%s" is tivo compatible' % fname)
 240                 f = open(fname, 'rb')
 241                 try:
 242                     if mime == 'video/mp4':
 243                         count = qtfaststart.process(f, handler.wfile, offset)
 244                     else:
 245                         if offset:
 246                             offset -= len(thead)
 247                             f.seek(offset)
 248                         while True:
 249                             block = f.read(512 * 1024)
 250                             if not block:
 251                                 break
 252                             handler.wfile.write(block)
 253                             count += len(block)
 254                 except Exception, msg:
 255                     logger.info(msg)
 256                 f.close()
 257             else:
 258                 logger.debug('"%s" is not tivo compatible' % fname)
 259                 if offset:
 260                     count = transcode.resume_transfer(path, handler.wfile,
 261                                                       offset)
 262                 else:
 263                     count = transcode.transcode(False, path, handler.wfile,
 264                                                 tsn, mime, thead)
 265         try:
 266             if not compatible:
 267                  handler.wfile.write('0\r\n\r\n')
 268             handler.wfile.flush()
 269         except Exception, msg:
 270             logger.info(msg)
 271
 272         mega_elapsed = (time.time() - start) * 1024 * 1024
 273         if mega_elapsed < 1:
 274             mega_elapsed = 1
 275         rate = count * 8.0 / mega_elapsed
 276         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 277                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 278                      tivo_name, count, rate))
 279
 280         if fname.endswith('.pyTivo-temp'):
 281             os.remove(fname)
 282
 283     def __duration(self, full_path):
 284         return transcode.video_info(full_path)['millisecs']
 285
 286     def __total_items(self, full_path):
 287         count = 0
 288         try:
 289             full_path = unicode(full_path, 'utf-8')
 290             for f in os.listdir(full_path):
 291                 if f.startswith('.'):
 292                     continue
 293                 f = os.path.join(full_path, f)
 294                 f2 = f.encode('utf-8')
 295                 if os.path.isdir(f):
 296                     count += 1
 297                 elif use_extensions:
 298                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 299                         count += 1
 300                 elif f2 in transcode.info_cache:
 301                     if transcode.supported_format(f2):
 302                         count += 1
 303         except:
 304             pass
 305         return count
 306
 307     def __est_size(self, full_path, tsn='', mime=''):
 308         # Size is estimated by taking audio and video bit rate adding 2%
 309
 310         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 311             return os.path.getsize(unicode(full_path, 'utf-8'))
 312         else:
 313             # Must be re-encoded
 314             audioBPS = config.getMaxAudioBR(tsn) * 1000
 315             #audioBPS = config.strtod(config.getAudioBR(tsn))
 316             videoBPS = transcode.select_videostr(full_path, tsn)
 317             bitrate =  audioBPS + videoBPS
 318             return int((self.__duration(full_path) / 1000) *
 319                        (bitrate * 1.02 / 8))
 320
 321     def metadata_full(self, full_path, tsn='', mime='', mtime=None):
 322         data = {}
 323         vInfo = transcode.video_info(full_path)
 324
 325         if ((int(vInfo['vHeight']) >= 720 and
 326              config.getTivoHeight >= 720) or
 327             (int(vInfo['vWidth']) >= 1280 and
 328              config.getTivoWidth >= 1280)):
 329             data['showingBits'] = '4096'
 330
 331         data.update(metadata.basic(full_path, mtime))
 332         if full_path[-5:].lower() == '.tivo':
 333             data.update(metadata.from_tivo(full_path))
 334         if full_path[-4:].lower() == '.wtv':
 335             data.update(metadata.from_mscore(vInfo['rawmeta']))
 336
 337         if 'episodeNumber' in data:
 338             try:
 339                 ep = int(data['episodeNumber'])
 340             except:
 341                 ep = 0
 342             data['episodeNumber'] = str(ep)
 343
 344         if config.getDebug() and 'vHost' not in data:
 345             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 346             if compatible:
 347                 transcode_options = {}
 348             else:
 349                 transcode_options = transcode.transcode(True, full_path,
 350                                                         '', tsn, mime)
 351             data['vHost'] = (
 352                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 353                 ['SOURCE INFO: '] +
 354                 ["%s=%s" % (k, v)
 355                  for k, v in sorted(vInfo.items(), reverse=True)] +
 356                 ['TRANSCODE OPTIONS: '] +
 357                 ["%s" % (v) for k, v in transcode_options.items()] +
 358                 ['SOURCE FILE: ', os.path.basename(full_path)]
 359             )
 360
 361         now = datetime.utcnow()
 362         if 'time' in data:
 363             if data['time'].lower() == 'file':
 364                 if not mtime:
 365                     mtime = os.path.getmtime(unicode(full_path, 'utf-8'))
 366                 try:
 367                     now = datetime.utcfromtimestamp(mtime)
 368                 except:
 369                     logger.warning('Bad file time on ' + full_path)
 370             elif data['time'].lower() == 'oad':
 371                     now = isodt(data['originalAirDate'])
 372             else:
 373                 try:
 374                     now = isodt(data['time'])
 375                 except:
 376                     logger.warning('Bad time format: ' + data['time'] +
 377                                    ' , using current time')
 378
 379         duration = self.__duration(full_path)
 380         duration_delta = timedelta(milliseconds = duration)
 381         min = duration_delta.seconds / 60
 382         sec = duration_delta.seconds % 60
 383         hours = min / 60
 384         min = min % 60
 385
 386         data.update({'time': now.isoformat(),
 387                      'startTime': now.isoformat(),
 388                      'stopTime': (now + duration_delta).isoformat(),
 389                      'size': self.__est_size(full_path, tsn, mime),
 390                      'duration': duration,
 391                      'iso_duration': ('P%sDT%sH%sM%sS' %
 392                           (duration_delta.days, hours, min, sec))})
 393
 394         return data
 395
 396     def QueryContainer(self, handler, query):
 397         tsn = handler.headers.getheader('tsn', '')
 398         subcname = query['Container'][0]
 399
 400         if not self.get_local_path(handler, query):
 401             handler.send_error(404)
 402             return
 403
 404         container = handler.container
 405         force_alpha = container.getboolean('force_alpha')
 406         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 407
 408         files, total, start = self.get_files(handler, query,
 409                                              self.video_file_filter,
 410                                              force_alpha)
 411
 412         videos = []
 413         local_base_path = self.get_local_base_path(handler, query)
 414         for f in files:
 415             video = VideoDetails()
 416             mtime = f.mdate
 417             try:
 418                 ltime = time.localtime(mtime)
 419             except:
 420                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 421                 mtime = time.time()
 422                 ltime = time.localtime(mtime)
 423             video['captureDate'] = hex(int(mtime))
 424             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 425             video['name'] = os.path.basename(f.name)
 426             video['path'] = f.name
 427             video['part_path'] = f.name.replace(local_base_path, '', 1)
 428             if not video['part_path'].startswith(os.path.sep):
 429                 video['part_path'] = os.path.sep + video['part_path']
 430             video['title'] = os.path.basename(f.name)
 431             video['is_dir'] = f.isdir
 432             if video['is_dir']:
 433                 video['small_path'] = subcname + '/' + video['name']
 434                 video['total_items'] = self.__total_items(f.name)
 435             else:
 436                 if len(files) == 1 or f.name in transcode.info_cache:
 437                     video['valid'] = transcode.supported_format(f.name)
 438                     if video['valid']:
 439                         video.update(self.metadata_full(f.name, tsn,
 440                                      mtime=mtime))
 441                         if len(files) == 1:
 442                             video['captureDate'] = hex(isogm(video['time']))
 443                 else:
 444                     video['valid'] = True
 445                     video.update(metadata.basic(f.name, mtime))
 446
 447                 if self.use_ts(tsn, f.name):
 448                     video['mime'] = 'video/x-tivo-mpeg-ts'
 449                 else:
 450                     video['mime'] = 'video/x-tivo-mpeg'
 451
 452                 video['textSize'] = metadata.human_size(f.size)
 453
 454             videos.append(video)
 455
 456         if use_html:
 457             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 458         else:
 459             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 460         t.container = handler.cname
 461         t.name = subcname
 462         t.total = total
 463         t.start = start
 464         t.videos = videos
 465         t.quote = quote
 466         t.escape = escape
 467         t.crc = zlib.crc32
 468         t.guid = config.getGUID()
 469         t.tivos = config.tivos
 470         t.tivo_names = config.tivo_names
 471         if use_html:
 472             handler.send_html(str(t))
 473         else:
 474             handler.send_xml(str(t))
 475
 476     def use_ts(self, tsn, file_path):
 477         if config.is_ts_capable(tsn):
 478             if file_path[-5:].lower() == '.tivo':
 479                 try:
 480                     flag = file(file_path).read(8)
 481                 except:
 482                     return False
 483                 if ord(flag[7]) & 0x20:
 484                     return True
 485             elif config.has_ts_flag():
 486                 return True
 487
 488         return False
 489
 490     def get_details_xml(self, tsn, file_path):
 491         if (tsn, file_path) in self.tvbus_cache:
 492             details = self.tvbus_cache[(tsn, file_path)]
 493         else:
 494             file_info = VideoDetails()
 495             file_info['valid'] = transcode.supported_format(file_path)
 496             if file_info['valid']:
 497                 file_info.update(self.metadata_full(file_path, tsn))
 498
 499             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 500             t.video = file_info
 501             t.escape = escape
 502             t.get_tv = metadata.get_tv
 503             t.get_mpaa = metadata.get_mpaa
 504             t.get_stars = metadata.get_stars
 505             details = str(t)
 506             self.tvbus_cache[(tsn, file_path)] = details
 507         return details
 508
 509     def tivo_header(self, tsn, path, mime):
 510         if mime == 'video/x-tivo-mpeg-ts':
 511             flag = 45
 512         else:
 513             flag = 13
 514         details = self.get_details_xml(tsn, path)
 515         ld = len(details)
 516         chunklen = ld * 2 + 44
 517         padding = 2048 - chunklen % 1024
 518
 519         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 520                                             padding + chunklen, 2),
 521                         struct.pack('>LLHH', ld + 16, ld, 1, 0),
 522                         details, '\0' * 4,
 523                         struct.pack('>LLHH', ld + 19, ld, 2, 0),
 524                         details, '\0' * padding])
 525
 526     def TVBusQuery(self, handler, query):
 527         tsn = handler.headers.getheader('tsn', '')
 528         f = query['File'][0]
 529         path = self.get_local_path(handler, query)
 530         file_path = os.path.normpath(path + '/' + f)
 531
 532         details = self.get_details_xml(tsn, file_path)
 533
 534         handler.send_xml(details)
 535
 536 class Video(BaseVideo, Pushable):
 537         pass
 538
 539 class VideoDetails(DictMixin):
 540
 541     def __init__(self, d=None):
 542         if d:
 543             self.d = d
 544         else:
 545             self.d = {}
 546
 547     def __getitem__(self, key):
 548         if key not in self.d:
 549             self.d[key] = self.default(key)
 550         return self.d[key]
 551
 552     def __contains__(self, key):
 553         return True
 554
 555     def __setitem__(self, key, value):
 556         self.d[key] = value
 557
 558     def __delitem__(self):
 559         del self.d[key]
 560
 561     def keys(self):
 562         return self.d.keys()
 563
 564     def __iter__(self):
 565         return self.d.__iter__()
 566
 567     def iteritems(self):
 568         return self.d.iteritems()
 569
 570     def default(self, key):
 571         defaults = {
 572             'showingBits' : '0',
 573             'displayMajorNumber' : '0',
 574             'displayMinorNumber' : '0',
 575             'isEpisode' : 'true',
 576             'colorCode' : ('COLOR', '4'),
 577             'showType' : ('SERIES', '5')
 578         }
 579         if key in defaults:
 580             return defaults[key]
 581         elif key.startswith('v'):
 582             return []
 583         else:
 584             return ''