plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 logger = logging.getLogger('pyTivo.video.video')
  25
  26 SCRIPTDIR = os.path.dirname(__file__)
  27
  28 CLASS_NAME = 'Video'
  29
  30 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  31
  32 # Preload the templates
  33 def tmpl(name):
  34     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  35
  36 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  37 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  38 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  39
  40 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  41 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  42 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  43 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  44 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  45 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  46 .wm .wmd .wtv .yuv""".split()
  47
  48 use_extensions = True
  49 try:
  50     assert(config.get_bin('ffmpeg'))
  51 except:
  52     use_extensions = False
  53
  54 queue = []  # Recordings to push
  55
  56 def uniso(iso):
  57     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  58
  59 def isodt(iso):
  60     return datetime(*uniso(iso)[:6])
  61
  62 def isogm(iso):
  63     return int(calendar.timegm(uniso(iso)))
  64
  65 class Pushable(object):
  66
  67     def push_one_file(self, f):
  68         file_info = VideoDetails()
  69         file_info['valid'] = transcode.supported_format(f['path'])
  70
  71         mime = 'video/mpeg'
  72         if config.isHDtivo(f['tsn']):
  73             for m in ['video/mp4', 'video/bif']:
  74                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  75                     mime = m
  76                     break
  77
  78             if (mime == 'video/mpeg' and
  79                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  80                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'])
  81                 if new_path:
  82                     mime = 'video/mp4'
  83                     f['name'] = new_path
  84
  85         if file_info['valid']:
  86             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
  87
  88         url = f['url'] + quote(f['name'])
  89
  90         title = file_info['seriesTitle']
  91         if not title:
  92             title = file_info['title']
  93
  94         source = file_info['seriesId']
  95         if not source:
  96             source = title
  97
  98         subtitle = file_info['episodeTitle']
  99         try:
 100             m = mind.getMind(f['tsn'])
 101             m.pushVideo(
 102                 tsn = f['tsn'],
 103                 url = url,
 104                 description = file_info['description'],
 105                 duration = file_info['duration'] / 1000,
 106                 size = file_info['size'],
 107                 title = title,
 108                 subtitle = subtitle,
 109                 source = source,
 110                 mime = mime,
 111                 tvrating = file_info['tvRating'])
 112         except Exception, msg:
 113             logger.error(msg)
 114
 115     def process_queue(self):
 116         while queue:
 117             time.sleep(5)
 118             item = queue.pop(0)
 119             self.push_one_file(item)
 120
 121     def readip(self):
 122         """ returns your external IP address by querying dyndns.org """
 123         f = urllib.urlopen('http://checkip.dyndns.org/')
 124         s = f.read()
 125         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 126         return m.group(0)
 127
 128     def Push(self, handler, query):
 129         try:
 130             tsn = query['tsn'][0]
 131         except:
 132             logger.error('Push requires a TiVo Service Number')
 133             handler.send_error(404)
 134             return
 135
 136         for key in config.tivos:
 137             if config.tivos[key]['name'] == tsn:
 138                 tsn = key
 139                 break
 140         tivo_name = config.tivos[tsn].get('name', tsn)
 141
 142         container = quote(query['Container'][0].split('/')[0])
 143         ip = config.get_ip(tsn)
 144         port = config.getPort()
 145
 146         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 147         if config.getIsExternal(tsn):
 148             exturl = config.get_server('externalurl')
 149             if exturl:
 150                 if not exturl.endswith('/'):
 151                     exturl += '/'
 152                 baseurl = exturl + container
 153             else:
 154                 ip = self.readip()
 155                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 156
 157         path = self.get_local_base_path(handler, query)
 158
 159         files = query.get('File', [])
 160         for f in files:
 161             file_path = os.path.normpath(path + '/' + f)
 162             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 163                           'url': baseurl})
 164             if len(queue) == 1:
 165                 thread.start_new_thread(Video.process_queue, (self,))
 166
 167             logger.info('[%s] Queued "%s" for Push to %s' %
 168                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 169                          unicode(file_path, 'utf-8'), tivo_name))
 170
 171         files = [unicode(f, 'utf-8') for f in files]
 172         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 173
 174 class BaseVideo(Plugin):
 175
 176     CONTENT_TYPE = 'x-container/tivo-videos'
 177
 178     tvbus_cache = LRUCache(1)
 179
 180     def video_file_filter(self, full_path, type=None):
 181         if os.path.isdir(unicode(full_path, 'utf-8')):
 182             return True
 183         if use_extensions:
 184             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 185         else:
 186             return transcode.supported_format(full_path)
 187
 188     def send_file(self, handler, path, query):
 189         mime = 'video/x-tivo-mpeg'
 190         tsn = handler.headers.getheader('tsn', '')
 191         if tsn:
 192             tivo_name = config.tivos[tsn].get('name', tsn)
 193         else:
 194             tivo_name = handler.address_string()
 195
 196         is_tivo_file = (path[-5:].lower() == '.tivo')
 197
 198         if 'Format' in query:
 199             mime = query['Format'][0]
 200
 201         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 202         compatible = (not needs_tivodecode and
 203                       transcode.tivo_compatible(path, tsn, mime)[0])
 204
 205         try:  # "bytes=XXX-"
 206             offset = int(handler.headers.getheader('Range')[6:-1])
 207         except:
 208             offset = 0
 209
 210         if needs_tivodecode:
 211             valid = bool(config.get_bin('tivodecode') and
 212                          config.get_server('tivo_mak'))
 213         else:
 214             valid = True
 215
 216         if valid and offset:
 217             valid = ((compatible and offset < os.path.getsize(path)) or
 218                      (not compatible and transcode.is_resumable(path, offset)))
 219
 220         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 221         faking = (mime == 'video/x-tivo-mpeg' and
 222                   not (is_tivo_file and compatible))
 223         fname = unicode(path, 'utf-8')
 224         thead = ''
 225         if faking:
 226             thead = self.tivo_header(tsn, path, mime)
 227         if compatible:
 228             size = os.path.getsize(fname) + len(thead)
 229             handler.send_response(200)
 230             handler.send_header('Content-Length', size - offset)
 231             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 232                                 (offset, size - offset - 1, size))
 233         else:
 234             handler.send_response(206)
 235             handler.send_header('Transfer-Encoding', 'chunked')
 236         handler.send_header('Content-Type', mime)
 237         handler.end_headers()
 238
 239         logger.info('[%s] Start sending "%s" to %s' %
 240                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 241         start = time.time()
 242         count = 0
 243
 244         if valid:
 245             if compatible:
 246                 if faking and not offset:
 247                     handler.wfile.write(thead)
 248                 logger.debug('"%s" is tivo compatible' % fname)
 249                 f = open(fname, 'rb')
 250                 try:
 251                     if mime == 'video/mp4':
 252                         count = qtfaststart.process(f, handler.wfile, offset)
 253                     else:
 254                         if offset:
 255                             offset -= len(thead)
 256                             f.seek(offset)
 257                         while True:
 258                             block = f.read(512 * 1024)
 259                             if not block:
 260                                 break
 261                             handler.wfile.write(block)
 262                             count += len(block)
 263                 except Exception, msg:
 264                     logger.info(msg)
 265                 f.close()
 266             else:
 267                 logger.debug('"%s" is not tivo compatible' % fname)
 268                 if offset:
 269                     count = transcode.resume_transfer(path, handler.wfile,
 270                                                       offset)
 271                 else:
 272                     count = transcode.transcode(False, path, handler.wfile,
 273                                                 tsn, mime, thead)
 274         try:
 275             if not compatible:
 276                  handler.wfile.write('0\r\n\r\n')
 277             handler.wfile.flush()
 278         except Exception, msg:
 279             logger.info(msg)
 280
 281         mega_elapsed = (time.time() - start) * 1024 * 1024
 282         if mega_elapsed < 1:
 283             mega_elapsed = 1
 284         rate = count * 8.0 / mega_elapsed
 285         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 286                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 287                      tivo_name, count, rate))
 288
 289         if fname.endswith('.pyTivo-temp'):
 290             os.remove(fname)
 291
 292     def __duration(self, full_path):
 293         return transcode.video_info(full_path)['millisecs']
 294
 295     def __total_items(self, full_path):
 296         count = 0
 297         try:
 298             full_path = unicode(full_path, 'utf-8')
 299             for f in os.listdir(full_path):
 300                 if f.startswith('.'):
 301                     continue
 302                 f = os.path.join(full_path, f)
 303                 f2 = f.encode('utf-8')
 304                 if os.path.isdir(f):
 305                     count += 1
 306                 elif use_extensions:
 307                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 308                         count += 1
 309                 elif f2 in transcode.info_cache:
 310                     if transcode.supported_format(f2):
 311                         count += 1
 312         except:
 313             pass
 314         return count
 315
 316     def __est_size(self, full_path, tsn='', mime=''):
 317         # Size is estimated by taking audio and video bit rate adding 2%
 318
 319         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 320             return os.path.getsize(unicode(full_path, 'utf-8'))
 321         else:
 322             # Must be re-encoded
 323             audioBPS = config.getMaxAudioBR(tsn) * 1000
 324             #audioBPS = config.strtod(config.getAudioBR(tsn))
 325             videoBPS = transcode.select_videostr(full_path, tsn)
 326             bitrate =  audioBPS + videoBPS
 327             return int((self.__duration(full_path) / 1000) *
 328                        (bitrate * 1.02 / 8))
 329
 330     def metadata_full(self, full_path, tsn='', mime='', mtime=None):
 331         data = {}
 332         vInfo = transcode.video_info(full_path)
 333
 334         if ((int(vInfo['vHeight']) >= 720 and
 335              config.getTivoHeight >= 720) or
 336             (int(vInfo['vWidth']) >= 1280 and
 337              config.getTivoWidth >= 1280)):
 338             data['showingBits'] = '4096'
 339
 340         data.update(metadata.basic(full_path, mtime))
 341         if full_path[-5:].lower() == '.tivo':
 342             data.update(metadata.from_tivo(full_path))
 343         if full_path[-4:].lower() == '.wtv':
 344             data.update(metadata.from_mscore(vInfo['rawmeta']))
 345
 346         if 'episodeNumber' in data:
 347             try:
 348                 ep = int(data['episodeNumber'])
 349             except:
 350                 ep = 0
 351             data['episodeNumber'] = str(ep)
 352
 353         if config.getDebug() and 'vHost' not in data:
 354             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 355             if compatible:
 356                 transcode_options = {}
 357             else:
 358                 transcode_options = transcode.transcode(True, full_path,
 359                                                         '', tsn, mime)
 360             data['vHost'] = (
 361                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 362                 ['SOURCE INFO: '] +
 363                 ["%s=%s" % (k, v)
 364                  for k, v in sorted(vInfo.items(), reverse=True)] +
 365                 ['TRANSCODE OPTIONS: '] +
 366                 ["%s" % (v) for k, v in transcode_options.items()] +
 367                 ['SOURCE FILE: ', os.path.basename(full_path)]
 368             )
 369
 370         now = datetime.utcnow()
 371         if 'time' in data:
 372             if data['time'].lower() == 'file':
 373                 if not mtime:
 374                     mtime = os.path.getmtime(unicode(full_path, 'utf-8'))
 375                 try:
 376                     now = datetime.utcfromtimestamp(mtime)
 377                 except:
 378                     logger.warning('Bad file time on ' + full_path)
 379             elif data['time'].lower() == 'oad':
 380                     now = isodt(data['originalAirDate'])
 381             else:
 382                 try:
 383                     now = isodt(data['time'])
 384                 except:
 385                     logger.warning('Bad time format: ' + data['time'] +
 386                                    ' , using current time')
 387
 388         duration = self.__duration(full_path)
 389         duration_delta = timedelta(milliseconds = duration)
 390         min = duration_delta.seconds / 60
 391         sec = duration_delta.seconds % 60
 392         hours = min / 60
 393         min = min % 60
 394
 395         data.update({'time': now.isoformat(),
 396                      'startTime': now.isoformat(),
 397                      'stopTime': (now + duration_delta).isoformat(),
 398                      'size': self.__est_size(full_path, tsn, mime),
 399                      'duration': duration,
 400                      'iso_duration': ('P%sDT%sH%sM%sS' %
 401                           (duration_delta.days, hours, min, sec))})
 402
 403         return data
 404
 405     def QueryContainer(self, handler, query):
 406         tsn = handler.headers.getheader('tsn', '')
 407         subcname = query['Container'][0]
 408
 409         if not self.get_local_path(handler, query):
 410             handler.send_error(404)
 411             return
 412
 413         container = handler.container
 414         force_alpha = container.getboolean('force_alpha')
 415         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 416
 417         files, total, start = self.get_files(handler, query,
 418                                              self.video_file_filter,
 419                                              force_alpha)
 420
 421         videos = []
 422         local_base_path = self.get_local_base_path(handler, query)
 423         for f in files:
 424             video = VideoDetails()
 425             mtime = f.mdate
 426             try:
 427                 ltime = time.localtime(mtime)
 428             except:
 429                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 430                 mtime = time.time()
 431                 ltime = time.localtime(mtime)
 432             video['captureDate'] = hex(int(mtime))
 433             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 434             video['name'] = os.path.basename(f.name)
 435             video['path'] = f.name
 436             video['part_path'] = f.name.replace(local_base_path, '', 1)
 437             if not video['part_path'].startswith(os.path.sep):
 438                 video['part_path'] = os.path.sep + video['part_path']
 439             video['title'] = os.path.basename(f.name)
 440             video['is_dir'] = f.isdir
 441             if video['is_dir']:
 442                 video['small_path'] = subcname + '/' + video['name']
 443                 video['total_items'] = self.__total_items(f.name)
 444             else:
 445                 if len(files) == 1 or f.name in transcode.info_cache:
 446                     video['valid'] = transcode.supported_format(f.name)
 447                     if video['valid']:
 448                         video.update(self.metadata_full(f.name, tsn,
 449                                      mtime=mtime))
 450                         if len(files) == 1:
 451                             video['captureDate'] = hex(isogm(video['time']))
 452                 else:
 453                     video['valid'] = True
 454                     video.update(metadata.basic(f.name, mtime))
 455
 456                 if self.use_ts(tsn, f.name):
 457                     video['mime'] = 'video/x-tivo-mpeg-ts'
 458                 else:
 459                     video['mime'] = 'video/x-tivo-mpeg'
 460
 461                 video['textSize'] = metadata.human_size(f.size)
 462
 463             videos.append(video)
 464
 465         if use_html:
 466             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 467         else:
 468             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 469         t.container = handler.cname
 470         t.name = subcname
 471         t.total = total
 472         t.start = start
 473         t.videos = videos
 474         t.quote = quote
 475         t.escape = escape
 476         t.crc = zlib.crc32
 477         t.guid = config.getGUID()
 478         t.tivos = config.tivos
 479         if use_html:
 480             handler.send_html(str(t))
 481         else:
 482             handler.send_xml(str(t))
 483
 484     def use_ts(self, tsn, file_path):
 485         if config.is_ts_capable(tsn):
 486             if file_path[-5:].lower() == '.tivo':
 487                 try:
 488                     flag = file(file_path).read(8)
 489                 except:
 490                     return False
 491                 if ord(flag[7]) & 0x20:
 492                     return True
 493             elif config.has_ts_flag():
 494                 return True
 495
 496         return False
 497
 498     def get_details_xml(self, tsn, file_path):
 499         if (tsn, file_path) in self.tvbus_cache:
 500             details = self.tvbus_cache[(tsn, file_path)]
 501         else:
 502             file_info = VideoDetails()
 503             file_info['valid'] = transcode.supported_format(file_path)
 504             if file_info['valid']:
 505                 file_info.update(self.metadata_full(file_path, tsn))
 506
 507             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 508             t.video = file_info
 509             t.escape = escape
 510             t.get_tv = metadata.get_tv
 511             t.get_mpaa = metadata.get_mpaa
 512             t.get_stars = metadata.get_stars
 513             details = str(t)
 514             self.tvbus_cache[(tsn, file_path)] = details
 515         return details
 516
 517     def tivo_header(self, tsn, path, mime):
 518         def pad(length, align):
 519             extra = length % align
 520             if extra:
 521                 extra = align - extra
 522             return extra
 523
 524         if mime == 'video/x-tivo-mpeg-ts':
 525             flag = 45
 526         else:
 527             flag = 13
 528         details = self.get_details_xml(tsn, path)
 529         ld = len(details)
 530         chunk = details + '\0' * (pad(ld, 4) + 4)
 531         lc = len(chunk)
 532         blocklen = lc * 2 + 40
 533         padding = pad(blocklen, 1024)
 534
 535         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 536                                             padding + blocklen, 2),
 537                         struct.pack('>LLHH', lc + 12, ld, 1, 0),
 538                         chunk,
 539                         struct.pack('>LLHH', lc + 12, ld, 2, 0),
 540                         chunk, '\0' * padding])
 541
 542     def TVBusQuery(self, handler, query):
 543         tsn = handler.headers.getheader('tsn', '')
 544         f = query['File'][0]
 545         path = self.get_local_path(handler, query)
 546         file_path = os.path.normpath(path + '/' + f)
 547
 548         details = self.get_details_xml(tsn, file_path)
 549
 550         handler.send_xml(details)
 551
 552 class Video(BaseVideo, Pushable):
 553         pass
 554
 555 class VideoDetails(DictMixin):
 556
 557     def __init__(self, d=None):
 558         if d:
 559             self.d = d
 560         else:
 561             self.d = {}
 562
 563     def __getitem__(self, key):
 564         if key not in self.d:
 565             self.d[key] = self.default(key)
 566         return self.d[key]
 567
 568     def __contains__(self, key):
 569         return True
 570
 571     def __setitem__(self, key, value):
 572         self.d[key] = value
 573
 574     def __delitem__(self):
 575         del self.d[key]
 576
 577     def keys(self):
 578         return self.d.keys()
 579
 580     def __iter__(self):
 581         return self.d.__iter__()
 582
 583     def iteritems(self):
 584         return self.d.iteritems()
 585
 586     def default(self, key):
 587         defaults = {
 588             'showingBits' : '0',
 589             'displayMajorNumber' : '0',
 590             'displayMinorNumber' : '0',
 591             'isEpisode' : 'true',
 592             'colorCode' : ('COLOR', '4'),
 593             'showType' : ('SERIES', '5')
 594         }
 595         if key in defaults:
 596             return defaults[key]
 597         elif key.startswith('v'):
 598             return []
 599         else:
 600             return ''