plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import thread
   6 import time
   7 import urllib2
   8 import urlparse
   9 from urllib import quote, unquote
  10 from xml.dom import minidom
  11 from xml.sax.saxutils import escape
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Default top-level share path
  32
  33 DEFPATH = '/TiVoConnect?Command=QueryContainer&Container=/NowPlaying'
  34
  35 # Some error/status message templates
  36
  37 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  38 "togo_path" before using this function.</p>"""
  39
  40 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  41 transfer to:</p> <p>%s</p>"""
  42
  43 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  44 <p>%s</p> <p>has been stopped.</p>"""
  45
  46 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  47 from the queue.</p>"""
  48
  49 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  50 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  51 incorrect Media Access Key. Please return to the Settings page and
  52 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  53
  54 # Preload the templates
  55 def tmpl(name):
  56     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  57
  58 CONTAINER_TEMPLATE_MOBILE = tmpl('npl_mob.tmpl')
  59 CONTAINER_TEMPLATE = tmpl('npl.tmpl')
  60
  61 status = {} # Global variable to control download threads
  62 tivo_cache = {} # Cache of TiVo NPL
  63 queue = {} # Recordings to download -- list per TiVo
  64 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  65
  66 def null_cookie(name, value):
  67     return cookielib.Cookie(0, name, value, None, False, '', False,
  68         False, '', False, False, None, False, None, None, None)
  69
  70 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  71 cj = cookielib.CookieJar()
  72 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  73 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  74                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  75                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  76
  77 class ToGo(Plugin):
  78     CONTENT_TYPE = 'text/html'
  79
  80     def tivo_open(self, url):
  81         # Loop just in case we get a server busy message
  82         while True:
  83             try:
  84                 # Open the URL using our authentication/cookie opener
  85                 return tivo_opener.open(url)
  86
  87             # Do a retry if the TiVo responds that the server is busy
  88             except urllib2.HTTPError, e:
  89                 if e.code == 503:
  90                     time.sleep(5)
  91                     continue
  92
  93                 # Log and throw the error otherwise
  94                 logger.error(e)
  95                 raise
  96
  97     def NPL(self, handler, query):
  98
  99         def getint(thing):
 100             try:
 101                 result = int(thing)
 102             except:
 103                 result = 0
 104             return result
 105
 106         global basic_meta
 107         shows_per_page = 50 # Change this to alter the number of shows returned
 108         folder = ''
 109         FirstAnchor = ''
 110         has_tivodecode = bool(config.get_bin('tivodecode'))
 111
 112         if 'TiVo' in query:
 113             tivoIP = query['TiVo'][0]
 114             tsn = config.tivos_by_ip(tivoIP)
 115
 116             useragent = handler.headers.getheader('User-Agent', '')
 117
 118             attrs = config.tivos[tsn]
 119             tivo_name = attrs.get('name', tivoIP)
 120             tivo_mak = config.get_tsn('tivo_mak', tsn)
 121
 122             protocol = attrs.get('protocol', 'https')
 123             ip_port = '%s:%d' % (tivoIP, attrs.get('port', 443))
 124             path = attrs.get('path', DEFPATH)
 125             if 'Folder' in query:
 126                 folder = query['Folder'][0]
 127                 path += '/' + folder
 128             theurl = '%s://%s%s&ItemCount=%d' % (protocol, ip_port,
 129                                                  path, shows_per_page)
 130             if 'AnchorItem' in query:
 131                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 132             if 'AnchorOffset' in query:
 133                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 134
 135             if (theurl not in tivo_cache or
 136                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 137                 # if page is not cached or old then retreive it
 138                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 139                 try:
 140                     page = self.tivo_open(theurl)
 141                 except IOError, e:
 142                     handler.redir(UNABLE % (tivoIP, e), 10)
 143                     return
 144                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 145                                       'thepage_time': time.time()}
 146                 page.close()
 147
 148             xmldoc = tivo_cache[theurl]['thepage']
 149             items = xmldoc.getElementsByTagName('Item')
 150             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 151             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 152             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 153             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 154             if items:
 155                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 156
 157             data = []
 158             for item in items:
 159                 entry = {}
 160                 entry['ContentType'] = tag_data(item, 'Details/ContentType')
 161                 for tag in ('CopyProtected', 'UniqueId'):
 162                     value = tag_data(item, 'Details/' + tag)
 163                     if value:
 164                         entry[tag] = value
 165                 if entry['ContentType'] == 'x-tivo-container/folder':
 166                     entry['Title'] = tag_data(item, 'Details/Title')
 167                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 168                     lc = tag_data(item, 'Details/LastCaptureDate')
 169                     if not lc:
 170                         lc = tag_data(item, 'Details/LastChangeDate')
 171                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 172                         time.localtime(int(lc, 16)))
 173                 else:
 174                     keys = {'Icon': 'Links/CustomIcon/Url',
 175                             'Url': 'Links/Content/Url',
 176                             'Details': 'Links/TiVoVideoDetails/Url',
 177                             'SourceSize': 'Details/SourceSize',
 178                             'Duration': 'Details/Duration',
 179                             'CaptureDate': 'Details/CaptureDate'}
 180                     for key in keys:
 181                         value = tag_data(item, keys[key])
 182                         if value:
 183                             entry[key] = value
 184
 185                     if 'SourceSize' in entry:
 186                         rawsize = entry['SourceSize']
 187                         entry['SourceSize'] = metadata.human_size(rawsize)
 188
 189                     if 'Duration' in entry:
 190                         dur = getint(entry['Duration']) / 1000
 191                         entry['Duration'] = ( '%d:%02d:%02d' %
 192                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 193
 194                     if 'CaptureDate' in entry:
 195                         entry['CaptureDate'] = time.strftime('%b %d, %Y',
 196                             time.localtime(int(entry['CaptureDate'], 16)))
 197
 198                     url = entry['Url']
 199                     if url in basic_meta:
 200                         entry.update(basic_meta[url])
 201                     else:
 202                         basic_data = metadata.from_container(item)
 203                         entry.update(basic_data)
 204                         basic_meta[url] = basic_data
 205
 206                 data.append(entry)
 207         else:
 208             data = []
 209             tivoIP = ''
 210             TotalItems = 0
 211             ItemStart = 0
 212             ItemCount = 0
 213             title = ''
 214
 215         if useragent.lower().find('mobile') > 0:
 216             t = Template(CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 217         else:
 218             t = Template(CONTAINER_TEMPLATE, filter=EncodeUnicode)
 219         t.escape = escape
 220         t.quote = quote
 221         t.folder = folder
 222         t.status = status
 223         if tivoIP in queue:
 224             t.queue = queue[tivoIP]
 225         t.has_tivodecode = has_tivodecode
 226         t.togo_mpegts = config.is_ts_capable(tsn)
 227         t.tname = tivo_name
 228         t.tivoIP = tivoIP
 229         t.container = handler.cname
 230         t.data = data
 231         t.len = len
 232         t.TotalItems = getint(TotalItems)
 233         t.ItemStart = getint(ItemStart)
 234         t.ItemCount = getint(ItemCount)
 235         t.FirstAnchor = quote(FirstAnchor)
 236         t.shows_per_page = shows_per_page
 237         t.title = title
 238         handler.send_html(str(t), refresh='300')
 239
 240     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 241         # global status
 242         status[url].update({'running': True, 'queued': False})
 243
 244         parse_url = urlparse.urlparse(url)
 245
 246         name = unquote(parse_url[2])[10:].split('.')
 247         try:
 248             id = unquote(parse_url[4]).split('id=')[1]
 249             name.insert(-1, ' - ' + id)
 250         except:
 251             pass
 252         ts = status[url]['ts_format']
 253         if status[url]['decode']:
 254             if ts:
 255                 name[-1] = 'ts'
 256             else:
 257                 name[-1] = 'mpg'
 258         else:
 259             if ts:
 260                 name.insert(-1, ' (TS)')
 261             else:
 262                 name.insert(-1, ' (PS)')
 263         name.insert(-1, '.')
 264         name = ''.join(name)
 265         for ch in BADCHAR:
 266             name = name.replace(ch, BADCHAR[ch])
 267         outfile = os.path.join(togo_path, name)
 268
 269         if status[url]['save']:
 270             meta = basic_meta[url]
 271             details_url = 'https://%s/TiVoVideoDetails?id=%s' % (tivoIP, id)
 272             try:
 273                 handle = self.tivo_open(details_url)
 274                 meta.update(metadata.from_details(handle.read()))
 275                 handle.close()
 276             except:
 277                 pass
 278             metafile = open(outfile + '.txt', 'w')
 279             metadata.dump(metafile, meta)
 280             metafile.close()
 281
 282         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 283         try:
 284             if status[url]['ts_format']:
 285                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 286             else:
 287                 handle = self.tivo_open(url)
 288         except Exception, msg:
 289             status[url]['running'] = False
 290             status[url]['error'] = str(msg)
 291             return
 292
 293         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 294
 295         logger.info('[%s] Start getting "%s" from %s' %
 296                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 297
 298         if status[url]['decode']:
 299             tivodecode_path = config.get_bin('tivodecode')
 300             tcmd = [tivodecode_path, '-m', mak, '-o', outfile, '-']
 301             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 302                                           bufsize=(512 * 1024))
 303             f = tivodecode.stdin
 304         else:
 305             f = open(outfile, 'wb')
 306         length = 0
 307         start_time = time.time()
 308         last_interval = start_time
 309         now = start_time
 310         try:
 311             while status[url]['running']:
 312                 output = handle.read(1024000)
 313                 if not output:
 314                     break
 315                 length += len(output)
 316                 f.write(output)
 317                 now = time.time()
 318                 elapsed = now - last_interval
 319                 if elapsed >= 5:
 320                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 321                         (elapsed * 1024 * 1024))
 322                     status[url]['size'] += length
 323                     length = 0
 324                     last_interval = now
 325             if status[url]['running']:
 326                 status[url]['finished'] = True
 327         except Exception, msg:
 328             status[url]['running'] = False
 329             logger.info(msg)
 330         handle.close()
 331         f.close()
 332         status[url]['size'] += length
 333         if status[url]['running']:
 334             mega_elapsed = (now - start_time) * 1024 * 1024
 335             if mega_elapsed < 1:
 336                 mega_elapsed = 1
 337             size = status[url]['size']
 338             rate = size * 8.0 / mega_elapsed
 339             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 340                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 341                          tivo_name, size, rate))
 342             status[url]['running'] = False
 343         else:
 344             os.remove(outfile)
 345             if status[url]['save']:
 346                 os.remove(outfile + '.txt')
 347             logger.info('[%s] Transfer of "%s" from %s aborted' %
 348                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 349                          tivo_name))
 350             del status[url]
 351
 352     def process_queue(self, tivoIP, mak, togo_path):
 353         while queue[tivoIP]:
 354             time.sleep(5)
 355             url = queue[tivoIP][0]
 356             self.get_tivo_file(tivoIP, url, mak, togo_path)
 357             queue[tivoIP].pop(0)
 358         del queue[tivoIP]
 359
 360     def ToGo(self, handler, query):
 361         togo_path = config.get_server('togo_path')
 362         for name, data in config.getShares():
 363             if togo_path == name:
 364                 togo_path = data.get('path')
 365         if togo_path:
 366             tivoIP = query['TiVo'][0]
 367             tsn = config.tivos_by_ip(tivoIP)
 368             tivo_mak = config.get_tsn('tivo_mak', tsn)
 369             urls = query.get('Url', [])
 370             decode = 'decode' in query
 371             save = 'save' in query
 372             ts_format = 'ts_format' in query
 373             for theurl in urls:
 374                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 375                                   'queued': True, 'size': 0, 'finished': False,
 376                                   'decode': decode, 'save': save,
 377                                   'ts_format': ts_format}
 378                 if tivoIP in queue:
 379                     queue[tivoIP].append(theurl)
 380                 else:
 381                     queue[tivoIP] = [theurl]
 382                     thread.start_new_thread(ToGo.process_queue,
 383                                             (self, tivoIP, tivo_mak, togo_path))
 384                 logger.info('[%s] Queued "%s" for transfer to %s' %
 385                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 386                              unquote(theurl), togo_path))
 387             urlstring = '<br>'.join([unquote(x) for x in urls])
 388             message = TRANS_QUEUE % (urlstring, togo_path)
 389         else:
 390             message = MISSING
 391         handler.redir(message, 5)
 392
 393     def ToGoStop(self, handler, query):
 394         theurl = query['Url'][0]
 395         status[theurl]['running'] = False
 396         handler.redir(TRANS_STOP % unquote(theurl))
 397
 398     def Unqueue(self, handler, query):
 399         theurl = query['Url'][0]
 400         tivoIP = query['TiVo'][0]
 401         del status[theurl]
 402         queue[tivoIP].remove(theurl)
 403         logger.info('[%s] Removed "%s" from queue' %
 404                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 405                      unquote(theurl)))
 406         handler.redir(UNQUEUE % unquote(theurl))