plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import sys
   6 import thread
   7 import time
   8 import urllib2
   9 import urlparse
  10 from urllib import quote, unquote
  11 from xml.dom import minidom
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Default top-level share path
  32
  33 DEFPATH = '/TiVoConnect?Command=QueryContainer&Container=/NowPlaying'
  34
  35 # Some error/status message templates
  36
  37 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  38 "togo_path" before using this function.</p>"""
  39
  40 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  41 transfer to:</p> <p>%s</p>"""
  42
  43 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  44 <p>%s</p> <p>has been stopped.</p>"""
  45
  46 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  47 from the queue.</p>"""
  48
  49 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  50 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  51 incorrect Media Access Key. Please return to the Settings page and
  52 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  53
  54 # Preload the templates
  55 tnname = os.path.join(SCRIPTDIR, 'templates', 'npl.tmpl')
  56 NPL_TEMPLATE = file(tnname, 'rb').read()
  57
  58 mswindows = (sys.platform == "win32")
  59
  60 status = {} # Global variable to control download threads
  61 tivo_cache = {} # Cache of TiVo NPL
  62 queue = {} # Recordings to download -- list per TiVo
  63 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  64 details_urls = {} # URLs for extended data, indexed by main URL
  65
  66 def null_cookie(name, value):
  67     return cookielib.Cookie(0, name, value, None, False, '', False,
  68         False, '', False, False, None, False, None, None, None)
  69
  70 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  71 cj = cookielib.CookieJar()
  72 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  73 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  74                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  75                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  76
  77 class ToGo(Plugin):
  78     CONTENT_TYPE = 'text/html'
  79
  80     def tivo_open(self, url):
  81         # Loop just in case we get a server busy message
  82         while True:
  83             try:
  84                 # Open the URL using our authentication/cookie opener
  85                 return tivo_opener.open(url)
  86
  87             # Do a retry if the TiVo responds that the server is busy
  88             except urllib2.HTTPError, e:
  89                 if e.code == 503:
  90                     time.sleep(5)
  91                     continue
  92
  93                 # Log and throw the error otherwise
  94                 logger.error(e)
  95                 raise
  96
  97     def NPL(self, handler, query):
  98
  99         def getint(thing):
 100             try:
 101                 result = int(thing)
 102             except:
 103                 result = 0
 104             return result
 105
 106         global basic_meta
 107         global details_urls
 108         shows_per_page = 50 # Change this to alter the number of shows returned
 109         folder = ''
 110         FirstAnchor = ''
 111         has_tivodecode = bool(config.get_bin('tivodecode'))
 112
 113         if 'TiVo' in query:
 114             tivoIP = query['TiVo'][0]
 115             tsn = config.tivos_by_ip(tivoIP)
 116             attrs = config.tivos[tsn]
 117             tivo_name = attrs.get('name', tivoIP)
 118             tivo_mak = config.get_tsn('tivo_mak', tsn)
 119
 120             protocol = attrs.get('protocol', 'https')
 121             ip_port = '%s:%d' % (tivoIP, attrs.get('port', 443))
 122             path = attrs.get('path', DEFPATH)
 123             baseurl = '%s://%s%s' % (protocol, ip_port, path)
 124             theurl = baseurl
 125             if 'Folder' in query:
 126                 folder = query['Folder'][0]
 127                 theurl = urlparse.urljoin(theurl, folder)
 128             theurl += '&ItemCount=%d' % shows_per_page
 129             if 'AnchorItem' in query:
 130                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 131             if 'AnchorOffset' in query:
 132                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 133
 134             if (theurl not in tivo_cache or
 135                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 136                 # if page is not cached or old then retreive it
 137                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 138                 try:
 139                     page = self.tivo_open(theurl)
 140                 except IOError, e:
 141                     handler.redir(UNABLE % (tivoIP, e), 10)
 142                     return
 143                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 144                                       'thepage_time': time.time()}
 145                 page.close()
 146
 147             xmldoc = tivo_cache[theurl]['thepage']
 148             items = xmldoc.getElementsByTagName('Item')
 149             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 150             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 151             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 152             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 153             if items:
 154                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 155
 156             data = []
 157             for item in items:
 158                 entry = {}
 159                 for tag in ('CopyProtected', 'ContentType'):
 160                     value = tag_data(item, 'Details/' + tag)
 161                     if value:
 162                         entry[tag] = value
 163                 if entry['ContentType'].startswith('x-tivo-container'):
 164                     entry['Url'] = tag_data(item, 'Links/Content/Url')
 165                     entry['Title'] = tag_data(item, 'Details/Title')
 166                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 167                     lc = tag_data(item, 'Details/LastCaptureDate')
 168                     if not lc:
 169                         lc = tag_data(item, 'Details/LastChangeDate')
 170                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 171                         time.localtime(int(lc, 16)))
 172                 else:
 173                     keys = {'Icon': 'Links/CustomIcon/Url',
 174                             'Url': 'Links/Content/Url',
 175                             'Details': 'Links/TiVoVideoDetails/Url',
 176                             'SourceSize': 'Details/SourceSize',
 177                             'Duration': 'Details/Duration',
 178                             'CaptureDate': 'Details/CaptureDate'}
 179                     for key in keys:
 180                         value = tag_data(item, keys[key])
 181                         if value:
 182                             entry[key] = value
 183
 184                     if 'SourceSize' in entry:
 185                         rawsize = entry['SourceSize']
 186                         entry['SourceSize'] = metadata.human_size(rawsize)
 187
 188                     if 'Duration' in entry:
 189                         dur = getint(entry['Duration']) / 1000
 190                         entry['Duration'] = ( '%d:%02d:%02d' %
 191                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 192
 193                     if 'CaptureDate' in entry:
 194                         entry['CaptureDate'] = time.strftime('%b %d, %Y',
 195                             time.localtime(int(entry['CaptureDate'], 16)))
 196
 197                     url = urlparse.urljoin(baseurl, entry['Url'])
 198                     entry['Url'] = url
 199                     if url in basic_meta:
 200                         entry.update(basic_meta[url])
 201                     else:
 202                         basic_data = metadata.from_container(item)
 203                         entry.update(basic_data)
 204                         basic_meta[url] = basic_data
 205                         if 'Details' in entry:
 206                             details_urls[url] = entry['Details']
 207
 208                 data.append(entry)
 209         else:
 210             data = []
 211             tivoIP = ''
 212             TotalItems = 0
 213             ItemStart = 0
 214             ItemCount = 0
 215             title = ''
 216
 217         t = Template(NPL_TEMPLATE, filter=EncodeUnicode)
 218         t.quote = quote
 219         t.folder = folder
 220         t.status = status
 221         if tivoIP in queue:
 222             t.queue = queue[tivoIP]
 223         t.has_tivodecode = has_tivodecode
 224         t.togo_mpegts = config.is_ts_capable(tsn)
 225         t.tname = tivo_name
 226         t.tivoIP = tivoIP
 227         t.container = handler.cname
 228         t.data = data
 229         t.len = len
 230         t.TotalItems = getint(TotalItems)
 231         t.ItemStart = getint(ItemStart)
 232         t.ItemCount = getint(ItemCount)
 233         t.FirstAnchor = quote(FirstAnchor)
 234         t.shows_per_page = shows_per_page
 235         t.title = title
 236         handler.send_html(str(t), refresh='300')
 237
 238     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 239         # global status
 240         status[url].update({'running': True, 'queued': False})
 241
 242         parse_url = urlparse.urlparse(url)
 243
 244         name = unicode(unquote(parse_url[2]), 'utf-8').split('/')[-1].split('.')
 245         try:
 246             id = unquote(parse_url[4]).split('id=')[1]
 247             name.insert(-1, ' - ' + id)
 248         except:
 249             pass
 250         ts = status[url]['ts_format']
 251         if status[url]['decode']:
 252             if ts:
 253                 name[-1] = 'ts'
 254             else:
 255                 name[-1] = 'mpg'
 256         else:
 257             if ts:
 258                 name.insert(-1, ' (TS)')
 259             else:
 260                 name.insert(-1, ' (PS)')
 261         name.insert(-1, '.')
 262         name = ''.join(name)
 263         for ch in BADCHAR:
 264             name = name.replace(ch, BADCHAR[ch])
 265         outfile = os.path.join(togo_path, name)
 266
 267         if status[url]['save']:
 268             meta = basic_meta[url]
 269             try:
 270                 handle = self.tivo_open(details_urls[url])
 271                 meta.update(metadata.from_details(handle.read()))
 272                 handle.close()
 273             except:
 274                 pass
 275             metafile = open(outfile + '.txt', 'w')
 276             metadata.dump(metafile, meta)
 277             metafile.close()
 278
 279         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 280         try:
 281             if status[url]['ts_format']:
 282                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 283             else:
 284                 handle = self.tivo_open(url)
 285         except Exception, msg:
 286             status[url]['running'] = False
 287             status[url]['error'] = str(msg)
 288             return
 289
 290         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 291
 292         logger.info('[%s] Start getting "%s" from %s' %
 293                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 294
 295         if status[url]['decode']:
 296             fname = outfile
 297             if mswindows:
 298                 fname = fname.encode('iso8859-1')
 299             tivodecode_path = config.get_bin('tivodecode')
 300             tcmd = [tivodecode_path, '-m', mak, '-o', fname, '-']
 301             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 302                                           bufsize=(512 * 1024))
 303             f = tivodecode.stdin
 304         else:
 305             f = open(outfile, 'wb')
 306         length = 0
 307         start_time = time.time()
 308         last_interval = start_time
 309         now = start_time
 310         try:
 311             while status[url]['running']:
 312                 output = handle.read(1024000)
 313                 if not output:
 314                     break
 315                 length += len(output)
 316                 f.write(output)
 317                 now = time.time()
 318                 elapsed = now - last_interval
 319                 if elapsed >= 5:
 320                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 321                         (elapsed * 1024 * 1024))
 322                     status[url]['size'] += length
 323                     length = 0
 324                     last_interval = now
 325             if status[url]['running']:
 326                 status[url]['finished'] = True
 327         except Exception, msg:
 328             status[url]['running'] = False
 329             logger.info(msg)
 330         handle.close()
 331         f.close()
 332         status[url]['size'] += length
 333         if status[url]['running']:
 334             mega_elapsed = (now - start_time) * 1024 * 1024
 335             if mega_elapsed < 1:
 336                 mega_elapsed = 1
 337             size = status[url]['size']
 338             rate = size * 8.0 / mega_elapsed
 339             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 340                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 341                          tivo_name, size, rate))
 342             status[url]['running'] = False
 343         else:
 344             os.remove(outfile)
 345             if status[url]['save']:
 346                 os.remove(outfile + '.txt')
 347             logger.info('[%s] Transfer of "%s" from %s aborted' %
 348                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 349                          tivo_name))
 350             del status[url]
 351
 352     def process_queue(self, tivoIP, mak, togo_path):
 353         while queue[tivoIP]:
 354             time.sleep(5)
 355             url = queue[tivoIP][0]
 356             self.get_tivo_file(tivoIP, url, mak, togo_path)
 357             queue[tivoIP].pop(0)
 358         del queue[tivoIP]
 359
 360     def ToGo(self, handler, query):
 361         togo_path = config.get_server('togo_path')
 362         for name, data in config.getShares():
 363             if togo_path == name:
 364                 togo_path = data.get('path')
 365         if togo_path:
 366             tivoIP = query['TiVo'][0]
 367             tsn = config.tivos_by_ip(tivoIP)
 368             tivo_mak = config.get_tsn('tivo_mak', tsn)
 369             urls = query.get('Url', [])
 370             decode = 'decode' in query
 371             save = 'save' in query
 372             ts_format = 'ts_format' in query
 373             for theurl in urls:
 374                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 375                                   'queued': True, 'size': 0, 'finished': False,
 376                                   'decode': decode, 'save': save,
 377                                   'ts_format': ts_format}
 378                 if tivoIP in queue:
 379                     queue[tivoIP].append(theurl)
 380                 else:
 381                     queue[tivoIP] = [theurl]
 382                     thread.start_new_thread(ToGo.process_queue,
 383                                             (self, tivoIP, tivo_mak, togo_path))
 384                 logger.info('[%s] Queued "%s" for transfer to %s' %
 385                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 386                              unquote(theurl), togo_path))
 387             urlstring = '<br>'.join([unicode(unquote(x), 'utf-8')
 388                                      for x in urls])
 389             message = TRANS_QUEUE % (urlstring, togo_path)
 390         else:
 391             message = MISSING
 392         handler.redir(message, 5)
 393
 394     def ToGoStop(self, handler, query):
 395         theurl = query['Url'][0]
 396         status[theurl]['running'] = False
 397         handler.redir(TRANS_STOP % unquote(theurl))
 398
 399     def Unqueue(self, handler, query):
 400         theurl = query['Url'][0]
 401         tivoIP = query['TiVo'][0]
 402         del status[theurl]
 403         queue[tivoIP].remove(theurl)
 404         logger.info('[%s] Removed "%s" from queue' %
 405                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 406                      unquote(theurl)))
 407         handler.redir(UNQUEUE % unquote(theurl))