plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import sys
   6 import thread
   7 import time
   8 import urllib2
   9 import urlparse
  10 from urllib import quote, unquote
  11 from xml.dom import minidom
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Default top-level share path
  32
  33 DEFPATH = '/TiVoConnect?Command=QueryContainer&Container=/NowPlaying'
  34
  35 # Some error/status message templates
  36
  37 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  38 "togo_path" before using this function.</p>"""
  39
  40 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  41 transfer to:</p> <p>%s</p>"""
  42
  43 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  44 <p>%s</p> <p>has been stopped.</p>"""
  45
  46 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  47 from the queue.</p>"""
  48
  49 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  50 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  51 incorrect Media Access Key. Please return to the Settings page and
  52 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  53
  54 # Preload the templates
  55 def tmpl(name):
  56     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  57
  58 CONTAINER_TEMPLATE_MOBILE = tmpl('npl_mob.tmpl')
  59 CONTAINER_TEMPLATE = tmpl('npl.tmpl')
  60
  61 mswindows = (sys.platform == "win32")
  62
  63 status = {} # Global variable to control download threads
  64 tivo_cache = {} # Cache of TiVo NPL
  65 queue = {} # Recordings to download -- list per TiVo
  66 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  67 details_urls = {} # URLs for extended data, indexed by main URL
  68
  69 def null_cookie(name, value):
  70     return cookielib.Cookie(0, name, value, None, False, '', False,
  71         False, '', False, False, None, False, None, None, None)
  72
  73 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  74 cj = cookielib.CookieJar()
  75 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  76 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  77                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  78                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  79
  80 tsn = config.get_server('togo_tsn')
  81 if tsn:
  82     tivo_opener.addheaders.append(('TSN', tsn))
  83
  84 class ToGo(Plugin):
  85     CONTENT_TYPE = 'text/html'
  86
  87     def tivo_open(self, url):
  88         # Loop just in case we get a server busy message
  89         while True:
  90             try:
  91                 # Open the URL using our authentication/cookie opener
  92                 return tivo_opener.open(url)
  93
  94             # Do a retry if the TiVo responds that the server is busy
  95             except urllib2.HTTPError, e:
  96                 if e.code == 503:
  97                     time.sleep(5)
  98                     continue
  99
 100                 # Log and throw the error otherwise
 101                 logger.error(e)
 102                 raise
 103
 104     def NPL(self, handler, query):
 105
 106         def getint(thing):
 107             try:
 108                 result = int(thing)
 109             except:
 110                 result = 0
 111             return result
 112
 113         global basic_meta
 114         global details_urls
 115         shows_per_page = 50 # Change this to alter the number of shows returned
 116         folder = ''
 117         FirstAnchor = ''
 118         has_tivodecode = bool(config.get_bin('tivodecode'))
 119
 120         if 'TiVo' in query:
 121             tivoIP = query['TiVo'][0]
 122             tsn = config.tivos_by_ip(tivoIP)
 123
 124             useragent = handler.headers.getheader('User-Agent', '')
 125
 126             attrs = config.tivos[tsn]
 127             tivo_name = attrs.get('name', tivoIP)
 128             tivo_mak = config.get_tsn('tivo_mak', tsn)
 129
 130             protocol = attrs.get('protocol', 'https')
 131             ip_port = '%s:%d' % (tivoIP, attrs.get('port', 443))
 132             path = attrs.get('path', DEFPATH)
 133             baseurl = '%s://%s%s' % (protocol, ip_port, path)
 134             theurl = baseurl
 135             if 'Folder' in query:
 136                 folder = query['Folder'][0]
 137                 theurl = urlparse.urljoin(theurl, folder)
 138             theurl += '&ItemCount=%d' % shows_per_page
 139             if 'AnchorItem' in query:
 140                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 141             if 'AnchorOffset' in query:
 142                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 143
 144             if (theurl not in tivo_cache or
 145                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 146                 # if page is not cached or old then retreive it
 147                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 148                 try:
 149                     page = self.tivo_open(theurl)
 150                 except IOError, e:
 151                     handler.redir(UNABLE % (tivoIP, e), 10)
 152                     return
 153                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 154                                       'thepage_time': time.time()}
 155                 page.close()
 156
 157             xmldoc = tivo_cache[theurl]['thepage']
 158             items = xmldoc.getElementsByTagName('Item')
 159             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 160             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 161             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 162             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 163             if items:
 164                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 165
 166             data = []
 167             for item in items:
 168                 entry = {}
 169                 for tag in ('CopyProtected', 'ContentType'):
 170                     value = tag_data(item, 'Details/' + tag)
 171                     if value:
 172                         entry[tag] = value
 173                 if entry['ContentType'].startswith('x-tivo-container'):
 174                     entry['Url'] = tag_data(item, 'Links/Content/Url')
 175                     entry['Title'] = tag_data(item, 'Details/Title')
 176                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 177                     lc = tag_data(item, 'Details/LastCaptureDate')
 178                     if not lc:
 179                         lc = tag_data(item, 'Details/LastChangeDate')
 180                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 181                         time.localtime(int(lc, 16)))
 182                 else:
 183                     keys = {'Icon': 'Links/CustomIcon/Url',
 184                             'Url': 'Links/Content/Url',
 185                             'Details': 'Links/TiVoVideoDetails/Url',
 186                             'SourceSize': 'Details/SourceSize',
 187                             'Duration': 'Details/Duration',
 188                             'CaptureDate': 'Details/CaptureDate'}
 189                     for key in keys:
 190                         value = tag_data(item, keys[key])
 191                         if value:
 192                             entry[key] = value
 193
 194                     if 'SourceSize' in entry:
 195                         rawsize = entry['SourceSize']
 196                         entry['SourceSize'] = metadata.human_size(rawsize)
 197
 198                     if 'Duration' in entry:
 199                         dur = getint(entry['Duration']) / 1000
 200                         entry['Duration'] = ( '%d:%02d:%02d' %
 201                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 202
 203                     if 'CaptureDate' in entry:
 204                         entry['CaptureDate'] = time.strftime('%b %d, %Y',
 205                             time.localtime(int(entry['CaptureDate'], 16)))
 206
 207                     url = urlparse.urljoin(baseurl, entry['Url'])
 208                     entry['Url'] = url
 209                     if url in basic_meta:
 210                         entry.update(basic_meta[url])
 211                     else:
 212                         basic_data = metadata.from_container(item)
 213                         entry.update(basic_data)
 214                         basic_meta[url] = basic_data
 215                         if 'Details' in entry:
 216                             details_urls[url] = entry['Details']
 217
 218                 data.append(entry)
 219         else:
 220             data = []
 221             tivoIP = ''
 222             TotalItems = 0
 223             ItemStart = 0
 224             ItemCount = 0
 225             title = ''
 226
 227         if useragent.lower().find('mobile') > 0:
 228             t = Template(CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 229         else:
 230             t = Template(CONTAINER_TEMPLATE, filter=EncodeUnicode)
 231         t.quote = quote
 232         t.folder = folder
 233         t.status = status
 234         if tivoIP in queue:
 235             t.queue = queue[tivoIP]
 236         t.has_tivodecode = has_tivodecode
 237         t.togo_mpegts = config.is_ts_capable(tsn)
 238         t.tname = tivo_name
 239         t.tivoIP = tivoIP
 240         t.container = handler.cname
 241         t.data = data
 242         t.len = len
 243         t.TotalItems = getint(TotalItems)
 244         t.ItemStart = getint(ItemStart)
 245         t.ItemCount = getint(ItemCount)
 246         t.FirstAnchor = quote(FirstAnchor)
 247         t.shows_per_page = shows_per_page
 248         t.title = title
 249         handler.send_html(str(t), refresh='300')
 250
 251     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 252         # global status
 253         status[url].update({'running': True, 'queued': False})
 254
 255         parse_url = urlparse.urlparse(url)
 256
 257         name = unicode(unquote(parse_url[2]), 'utf-8').split('/')[-1].split('.')
 258         try:
 259             id = unquote(parse_url[4]).split('id=')[1]
 260             name.insert(-1, ' - ' + id)
 261         except:
 262             pass
 263         ts = status[url]['ts_format']
 264         if status[url]['decode']:
 265             if ts:
 266                 name[-1] = 'ts'
 267             else:
 268                 name[-1] = 'mpg'
 269         else:
 270             if ts:
 271                 name.insert(-1, ' (TS)')
 272             else:
 273                 name.insert(-1, ' (PS)')
 274         name.insert(-1, '.')
 275         name = ''.join(name)
 276         for ch in BADCHAR:
 277             name = name.replace(ch, BADCHAR[ch])
 278         outfile = os.path.join(togo_path, name)
 279
 280         if status[url]['save']:
 281             meta = basic_meta[url]
 282             try:
 283                 handle = self.tivo_open(details_urls[url])
 284                 meta.update(metadata.from_details(handle.read()))
 285                 handle.close()
 286             except:
 287                 pass
 288             metafile = open(outfile + '.txt', 'w')
 289             metadata.dump(metafile, meta)
 290             metafile.close()
 291
 292         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 293         try:
 294             if status[url]['ts_format']:
 295                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 296             else:
 297                 handle = self.tivo_open(url)
 298         except Exception, msg:
 299             status[url]['running'] = False
 300             status[url]['error'] = str(msg)
 301             return
 302
 303         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 304
 305         logger.info('[%s] Start getting "%s" from %s' %
 306                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 307
 308         if status[url]['decode']:
 309             fname = outfile
 310             if mswindows:
 311                 fname = fname.encode('cp1252')
 312             tivodecode_path = config.get_bin('tivodecode')
 313             tcmd = [tivodecode_path, '-m', mak, '-o', fname, '-']
 314             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 315                                           bufsize=(512 * 1024))
 316             f = tivodecode.stdin
 317         else:
 318             f = open(outfile, 'wb')
 319         length = 0
 320         start_time = time.time()
 321         last_interval = start_time
 322         now = start_time
 323         try:
 324             while status[url]['running']:
 325                 output = handle.read(1024000)
 326                 if not output:
 327                     break
 328                 length += len(output)
 329                 f.write(output)
 330                 now = time.time()
 331                 elapsed = now - last_interval
 332                 if elapsed >= 5:
 333                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 334                         (elapsed * 1024 * 1024))
 335                     status[url]['size'] += length
 336                     length = 0
 337                     last_interval = now
 338             if status[url]['running']:
 339                 status[url]['finished'] = True
 340         except Exception, msg:
 341             status[url]['running'] = False
 342             logger.info(msg)
 343         handle.close()
 344         f.close()
 345         status[url]['size'] += length
 346         if status[url]['running']:
 347             mega_elapsed = (now - start_time) * 1024 * 1024
 348             if mega_elapsed < 1:
 349                 mega_elapsed = 1
 350             size = status[url]['size']
 351             rate = size * 8.0 / mega_elapsed
 352             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 353                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 354                          tivo_name, size, rate))
 355             status[url]['running'] = False
 356         else:
 357             os.remove(outfile)
 358             if status[url]['save']:
 359                 os.remove(outfile + '.txt')
 360             logger.info('[%s] Transfer of "%s" from %s aborted' %
 361                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 362                          tivo_name))
 363             del status[url]
 364
 365     def process_queue(self, tivoIP, mak, togo_path):
 366         while queue[tivoIP]:
 367             time.sleep(5)
 368             url = queue[tivoIP][0]
 369             self.get_tivo_file(tivoIP, url, mak, togo_path)
 370             queue[tivoIP].pop(0)
 371         del queue[tivoIP]
 372
 373     def ToGo(self, handler, query):
 374         togo_path = config.get_server('togo_path')
 375         for name, data in config.getShares():
 376             if togo_path == name:
 377                 togo_path = data.get('path')
 378         if togo_path:
 379             tivoIP = query['TiVo'][0]
 380             tsn = config.tivos_by_ip(tivoIP)
 381             tivo_mak = config.get_tsn('tivo_mak', tsn)
 382             urls = query.get('Url', [])
 383             decode = 'decode' in query
 384             save = 'save' in query
 385             ts_format = 'ts_format' in query
 386             for theurl in urls:
 387                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 388                                   'queued': True, 'size': 0, 'finished': False,
 389                                   'decode': decode, 'save': save,
 390                                   'ts_format': ts_format}
 391                 if tivoIP in queue:
 392                     queue[tivoIP].append(theurl)
 393                 else:
 394                     queue[tivoIP] = [theurl]
 395                     thread.start_new_thread(ToGo.process_queue,
 396                                             (self, tivoIP, tivo_mak, togo_path))
 397                 logger.info('[%s] Queued "%s" for transfer to %s' %
 398                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 399                              unquote(theurl), togo_path))
 400             urlstring = '<br>'.join([unicode(unquote(x), 'utf-8')
 401                                      for x in urls])
 402             message = TRANS_QUEUE % (urlstring, togo_path)
 403         else:
 404             message = MISSING
 405         handler.redir(message, 5)
 406
 407     def ToGoStop(self, handler, query):
 408         theurl = query['Url'][0]
 409         status[theurl]['running'] = False
 410         handler.redir(TRANS_STOP % unquote(theurl))
 411
 412     def Unqueue(self, handler, query):
 413         theurl = query['Url'][0]
 414         tivoIP = query['TiVo'][0]
 415         del status[theurl]
 416         queue[tivoIP].remove(theurl)
 417         logger.info('[%s] Removed "%s" from queue' %
 418                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 419                      unquote(theurl)))
 420         handler.redir(UNQUEUE % unquote(theurl))