plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 logger = logging.getLogger('pyTivo.video.video')
  25
  26 SCRIPTDIR = os.path.dirname(__file__)
  27
  28 CLASS_NAME = 'Video'
  29
  30 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  31
  32 # Preload the templates
  33 def tmpl(name):
  34     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  35
  36 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  37 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  38 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  39
  40 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  41 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  42 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  43 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  44 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  45 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  46 .wm .wmd .wtv .yuv""".split()
  47
  48 use_extensions = True
  49 try:
  50     assert(config.get_bin('ffmpeg'))
  51 except:
  52     use_extensions = False
  53
  54 queue = []  # Recordings to push
  55
  56 def uniso(iso):
  57     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  58
  59 def isodt(iso):
  60     return datetime(*uniso(iso)[:6])
  61
  62 def isogm(iso):
  63     return int(calendar.timegm(uniso(iso)))
  64
  65 class Pushable(object):
  66
  67     def push_one_file(self, f):
  68         file_info = VideoDetails()
  69         file_info['valid'] = transcode.supported_format(f['path'])
  70
  71         mime = 'video/mpeg'
  72         if config.isHDtivo(f['tsn']):
  73             for m in ['video/mp4', 'video/bif']:
  74                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  75                     mime = m
  76                     break
  77
  78             if (mime == 'video/mpeg' and
  79                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  80                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'])
  81                 if new_path:
  82                     mime = 'video/mp4'
  83                     f['name'] = new_path
  84
  85         if file_info['valid']:
  86             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
  87
  88         url = f['url'] + quote(f['name'])
  89
  90         title = file_info['seriesTitle']
  91         if not title:
  92             title = file_info['title']
  93
  94         source = file_info['seriesId']
  95         if not source:
  96             source = title
  97
  98         subtitle = file_info['episodeTitle']
  99         try:
 100             m = mind.getMind(f['tsn'])
 101             m.pushVideo(
 102                 tsn = f['tsn'],
 103                 url = url,
 104                 description = file_info['description'],
 105                 duration = file_info['duration'] / 1000,
 106                 size = file_info['size'],
 107                 title = title,
 108                 subtitle = subtitle,
 109                 source = source,
 110                 mime = mime,
 111                 tvrating = file_info['tvRating'])
 112         except Exception, msg:
 113             logger.error(msg)
 114
 115     def process_queue(self):
 116         while queue:
 117             time.sleep(5)
 118             item = queue.pop(0)
 119             self.push_one_file(item)
 120
 121     def readip(self):
 122         """ returns your external IP address by querying dyndns.org """
 123         f = urllib.urlopen('http://checkip.dyndns.org/')
 124         s = f.read()
 125         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 126         return m.group(0)
 127
 128     def Push(self, handler, query):
 129         tsn = query['tsn'][0]
 130         for key in config.tivo_names:
 131             if config.tivo_names[key] == tsn:
 132                 tsn = key
 133                 break
 134         tivo_name = config.tivo_names.get(tsn, tsn)
 135
 136         container = quote(query['Container'][0].split('/')[0])
 137         ip = config.get_ip(tsn)
 138         port = config.getPort()
 139
 140         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 141         if config.getIsExternal(tsn):
 142             exturl = config.get_server('externalurl')
 143             if exturl:
 144                 if not exturl.endswith('/'):
 145                     exturl += '/'
 146                 baseurl = exturl + container
 147             else:
 148                 ip = self.readip()
 149                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 150
 151         path = self.get_local_base_path(handler, query)
 152
 153         files = query.get('File', [])
 154         for f in files:
 155             file_path = os.path.normpath(path + f)
 156             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 157                           'url': baseurl})
 158             if len(queue) == 1:
 159                 thread.start_new_thread(Video.process_queue, (self,))
 160
 161             logger.info('[%s] Queued "%s" for Push to %s' %
 162                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 163                          unicode(file_path, 'utf-8'), tivo_name))
 164
 165         files = [unicode(f, 'utf-8') for f in files]
 166         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 167
 168 class BaseVideo(Plugin):
 169
 170     CONTENT_TYPE = 'x-container/tivo-videos'
 171
 172     tvbus_cache = LRUCache(1)
 173
 174     def video_file_filter(self, full_path, type=None):
 175         if os.path.isdir(unicode(full_path, 'utf-8')):
 176             return True
 177         if use_extensions:
 178             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 179         else:
 180             return transcode.supported_format(full_path)
 181
 182     def send_file(self, handler, path, query):
 183         mime = 'video/x-tivo-mpeg'
 184         tsn = handler.headers.getheader('tsn', '')
 185         tivo_name = config.tivo_names.get(tsn, tsn)
 186
 187         is_tivo_file = (path[-5:].lower() == '.tivo')
 188
 189         if 'Format' in query:
 190             mime = query['Format'][0]
 191
 192         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 193         compatible = (not needs_tivodecode and
 194                       transcode.tivo_compatible(path, tsn, mime)[0])
 195
 196         try:  # "bytes=XXX-"
 197             offset = int(handler.headers.getheader('Range')[6:-1])
 198         except:
 199             offset = 0
 200
 201         if needs_tivodecode:
 202             valid = bool(config.get_bin('tivodecode') and
 203                          config.get_server('tivo_mak'))
 204         else:
 205             valid = True
 206
 207         if valid and offset:
 208             valid = ((compatible and offset < os.stat(path).st_size) or
 209                      (not compatible and transcode.is_resumable(path, offset)))
 210
 211         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 212         faking = (mime == 'video/x-tivo-mpeg' and
 213                   not (is_tivo_file and compatible))
 214         fname = unicode(path, 'utf-8')
 215         thead = ''
 216         if faking:
 217             thead = self.tivo_header(tsn, path, mime)
 218         if compatible:
 219             size = os.stat(fname).st_size + len(thead)
 220             handler.send_response(200)
 221             handler.send_header('Content-Length', size - offset)
 222             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 223                                 (offset, size - offset - 1, size))
 224         else:
 225             handler.send_response(206)
 226             handler.send_header('Transfer-Encoding', 'chunked')
 227         handler.send_header('Content-Type', mime)
 228         handler.end_headers()
 229
 230         logger.info('[%s] Start sending "%s" to %s' %
 231                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 232         start = time.time()
 233         count = 0
 234
 235         if valid:
 236             if compatible:
 237                 if faking and not offset:
 238                     handler.wfile.write(thead)
 239                 logger.debug('"%s" is tivo compatible' % fname)
 240                 f = open(fname, 'rb')
 241                 try:
 242                     if mime == 'video/mp4':
 243                         count = qtfaststart.process(f, handler.wfile, offset)
 244                     else:
 245                         if offset:
 246                             offset -= len(thead)
 247                             f.seek(offset)
 248                         while True:
 249                             block = f.read(512 * 1024)
 250                             if not block:
 251                                 break
 252                             handler.wfile.write(block)
 253                             count += len(block)
 254                 except Exception, msg:
 255                     logger.info(msg)
 256                 f.close()
 257             else:
 258                 logger.debug('"%s" is not tivo compatible' % fname)
 259                 if offset:
 260                     count = transcode.resume_transfer(path, handler.wfile,
 261                                                       offset)
 262                 else:
 263                     count = transcode.transcode(False, path, handler.wfile,
 264                                                 tsn, mime, thead)
 265         try:
 266             if not compatible:
 267                  handler.wfile.write('0\r\n\r\n')
 268             handler.wfile.flush()
 269         except Exception, msg:
 270             logger.info(msg)
 271
 272         mega_elapsed = (time.time() - start) * 1024 * 1024
 273         if mega_elapsed < 1:
 274             mega_elapsed = 1
 275         rate = count * 8.0 / mega_elapsed
 276         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 277                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 278                      tivo_name, count, rate))
 279
 280         if fname.endswith('.pyTivo-temp'):
 281             os.remove(fname)
 282
 283     def __duration(self, full_path):
 284         return transcode.video_info(full_path)['millisecs']
 285
 286     def __total_items(self, full_path):
 287         count = 0
 288         try:
 289             full_path = unicode(full_path, 'utf-8')
 290             for f in os.listdir(full_path):
 291                 if f.startswith('.'):
 292                     continue
 293                 f = os.path.join(full_path, f)
 294                 f2 = f.encode('utf-8')
 295                 if os.path.isdir(f):
 296                     count += 1
 297                 elif use_extensions:
 298                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 299                         count += 1
 300                 elif f2 in transcode.info_cache:
 301                     if transcode.supported_format(f2):
 302                         count += 1
 303         except:
 304             pass
 305         return count
 306
 307     def __est_size(self, full_path, tsn='', mime=''):
 308         # Size is estimated by taking audio and video bit rate adding 2%
 309
 310         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 311             return int(os.stat(unicode(full_path, 'utf-8')).st_size)
 312         else:
 313             # Must be re-encoded
 314             audioBPS = config.getMaxAudioBR(tsn) * 1000
 315             #audioBPS = config.strtod(config.getAudioBR(tsn))
 316             videoBPS = transcode.select_videostr(full_path, tsn)
 317             bitrate =  audioBPS + videoBPS
 318             return int((self.__duration(full_path) / 1000) *
 319                        (bitrate * 1.02 / 8))
 320
 321     def metadata_full(self, full_path, tsn='', mime=''):
 322         data = {}
 323         vInfo = transcode.video_info(full_path)
 324
 325         if ((int(vInfo['vHeight']) >= 720 and
 326              config.getTivoHeight >= 720) or
 327             (int(vInfo['vWidth']) >= 1280 and
 328              config.getTivoWidth >= 1280)):
 329             data['showingBits'] = '4096'
 330
 331         data.update(metadata.basic(full_path))
 332         if full_path[-5:].lower() == '.tivo':
 333             data.update(metadata.from_tivo(full_path))
 334         if full_path[-4:].lower() == '.wtv':
 335             data.update(metadata.from_mscore(vInfo['rawmeta']))
 336
 337         if 'episodeNumber' in data:
 338             try:
 339                 ep = int(data['episodeNumber'])
 340             except:
 341                 ep = 0
 342             data['episodeNumber'] = str(ep)
 343
 344         if config.getDebug() and 'vHost' not in data:
 345             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 346             if compatible:
 347                 transcode_options = {}
 348             else:
 349                 transcode_options = transcode.transcode(True, full_path,
 350                                                         '', tsn, mime)
 351             data['vHost'] = (
 352                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 353                 ['SOURCE INFO: '] +
 354                 ["%s=%s" % (k, v)
 355                  for k, v in sorted(vInfo.items(), reverse=True)] +
 356                 ['TRANSCODE OPTIONS: '] +
 357                 ["%s" % (v) for k, v in transcode_options.items()] +
 358                 ['SOURCE FILE: ', os.path.basename(full_path)]
 359             )
 360
 361         now = datetime.utcnow()
 362         if 'time' in data:
 363             if data['time'].lower() == 'file':
 364                 mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 365                 try:
 366                     now = datetime.utcfromtimestamp(mtime)
 367                 except:
 368                     logger.warning('Bad file time on ' + full_path)
 369             elif data['time'].lower() == 'oad':
 370                     now = isodt(data['originalAirDate'])
 371             else:
 372                 try:
 373                     now = isodt(data['time'])
 374                 except:
 375                     logger.warning('Bad time format: ' + data['time'] +
 376                                    ' , using current time')
 377
 378         duration = self.__duration(full_path)
 379         duration_delta = timedelta(milliseconds = duration)
 380         min = duration_delta.seconds / 60
 381         sec = duration_delta.seconds % 60
 382         hours = min / 60
 383         min = min % 60
 384
 385         data.update({'time': now.isoformat(),
 386                      'startTime': now.isoformat(),
 387                      'stopTime': (now + duration_delta).isoformat(),
 388                      'size': self.__est_size(full_path, tsn, mime),
 389                      'duration': duration,
 390                      'iso_duration': ('P%sDT%sH%sM%sS' %
 391                           (duration_delta.days, hours, min, sec))})
 392
 393         return data
 394
 395     def QueryContainer(self, handler, query):
 396         tsn = handler.headers.getheader('tsn', '')
 397         subcname = query['Container'][0]
 398
 399         if not self.get_local_path(handler, query):
 400             handler.send_error(404)
 401             return
 402
 403         container = handler.container
 404         force_alpha = container.getboolean('force_alpha')
 405         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 406
 407         files, total, start = self.get_files(handler, query,
 408                                              self.video_file_filter,
 409                                              force_alpha)
 410
 411         videos = []
 412         local_base_path = self.get_local_base_path(handler, query)
 413         for f in files:
 414             video = VideoDetails()
 415             mtime = f.mdate
 416             try:
 417                 ltime = time.localtime(mtime)
 418             except:
 419                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 420                 mtime = int(time.time())
 421                 ltime = time.localtime(mtime)
 422             video['captureDate'] = hex(mtime)
 423             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 424             video['name'] = os.path.basename(f.name)
 425             video['path'] = f.name
 426             video['part_path'] = f.name.replace(local_base_path, '', 1)
 427             if not video['part_path'].startswith(os.path.sep):
 428                 video['part_path'] = os.path.sep + video['part_path']
 429             video['title'] = os.path.basename(f.name)
 430             video['is_dir'] = f.isdir
 431             if video['is_dir']:
 432                 video['small_path'] = subcname + '/' + video['name']
 433                 video['total_items'] = self.__total_items(f.name)
 434             else:
 435                 if len(files) == 1 or f.name in transcode.info_cache:
 436                     video['valid'] = transcode.supported_format(f.name)
 437                     if video['valid']:
 438                         video.update(self.metadata_full(f.name, tsn))
 439                         if len(files) == 1:
 440                             video['captureDate'] = hex(isogm(video['time']))
 441                 else:
 442                     video['valid'] = True
 443                     video.update(metadata.basic(f.name))
 444
 445                 if self.use_ts(tsn, f.name):
 446                     video['mime'] = 'video/x-tivo-mpeg-ts'
 447                 else:
 448                     video['mime'] = 'video/x-tivo-mpeg'
 449
 450                 video['textSize'] = metadata.human_size(f.size)
 451
 452             videos.append(video)
 453
 454         if use_html:
 455             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 456         else:
 457             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 458         t.container = handler.cname
 459         t.name = subcname
 460         t.total = total
 461         t.start = start
 462         t.videos = videos
 463         t.quote = quote
 464         t.escape = escape
 465         t.crc = zlib.crc32
 466         t.guid = config.getGUID()
 467         t.tivos = config.tivos
 468         t.tivo_names = config.tivo_names
 469         if use_html:
 470             handler.send_html(str(t))
 471         else:
 472             handler.send_xml(str(t))
 473
 474     def use_ts(self, tsn, file_path):
 475         if config.is_ts_capable(tsn):
 476             if file_path[-5:].lower() == '.tivo':
 477                 try:
 478                     flag = file(file_path).read(8)
 479                 except:
 480                     return False
 481                 if ord(flag[7]) & 0x20:
 482                     return True
 483             elif config.has_ts_flag():
 484                 return True
 485
 486         return False
 487
 488     def get_details_xml(self, tsn, file_path):
 489         if (tsn, file_path) in self.tvbus_cache:
 490             details = self.tvbus_cache[(tsn, file_path)]
 491         else:
 492             file_info = VideoDetails()
 493             file_info['valid'] = transcode.supported_format(file_path)
 494             if file_info['valid']:
 495                 file_info.update(self.metadata_full(file_path, tsn))
 496
 497             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 498             t.video = file_info
 499             t.escape = escape
 500             t.get_tv = metadata.get_tv
 501             t.get_mpaa = metadata.get_mpaa
 502             t.get_stars = metadata.get_stars
 503             details = str(t)
 504             self.tvbus_cache[(tsn, file_path)] = details
 505         return details
 506
 507     def tivo_header(self, tsn, path, mime):
 508         if mime == 'video/x-tivo-mpeg-ts':
 509             flag = 45
 510         else:
 511             flag = 13
 512         details = self.get_details_xml(tsn, path)
 513         ld = len(details)
 514         chunklen = ld * 2 + 44
 515         padding = 2048 - chunklen % 1024
 516
 517         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 518                                             padding + chunklen, 2),
 519                         struct.pack('>LLHH', ld + 16, ld, 1, 0),
 520                         details, '\0' * 4,
 521                         struct.pack('>LLHH', ld + 19, ld, 2, 0),
 522                         details, '\0' * padding])
 523
 524     def TVBusQuery(self, handler, query):
 525         tsn = handler.headers.getheader('tsn', '')
 526         f = query['File'][0]
 527         path = self.get_local_path(handler, query)
 528         file_path = os.path.normpath(path + f)
 529
 530         details = self.get_details_xml(tsn, file_path)
 531
 532         handler.send_xml(details)
 533
 534 class Video(BaseVideo, Pushable):
 535         pass
 536
 537 class VideoDetails(DictMixin):
 538
 539     def __init__(self, d=None):
 540         if d:
 541             self.d = d
 542         else:
 543             self.d = {}
 544
 545     def __getitem__(self, key):
 546         if key not in self.d:
 547             self.d[key] = self.default(key)
 548         return self.d[key]
 549
 550     def __contains__(self, key):
 551         return True
 552
 553     def __setitem__(self, key, value):
 554         self.d[key] = value
 555
 556     def __delitem__(self):
 557         del self.d[key]
 558
 559     def keys(self):
 560         return self.d.keys()
 561
 562     def __iter__(self):
 563         return self.d.__iter__()
 564
 565     def iteritems(self):
 566         return self.d.iteritems()
 567
 568     def default(self, key):
 569         defaults = {
 570             'showingBits' : '0',
 571             'displayMajorNumber' : '0',
 572             'displayMinorNumber' : '0',
 573             'isEpisode' : 'true',
 574             'colorCode' : ('COLOR', '4'),
 575             'showType' : ('SERIES', '5')
 576         }
 577         if key in defaults:
 578             return defaults[key]
 579         elif key.startswith('v'):
 580             return []
 581         else:
 582             return ''