plugins/togo/togo.py

   1 import cgi
   2 import cookielib
   3 import logging
   4 import os
   5 import subprocess
   6 import sys
   7 import thread
   8 import time
   9 import urllib2
  10 import urlparse
  11 from urllib import quote, unquote
  12 from xml.dom import minidom
  13
  14 from Cheetah.Template import Template
  15
  16 import config
  17 import metadata
  18 from plugin import EncodeUnicode, Plugin
  19
  20 logger = logging.getLogger('pyTivo.togo')
  21 tag_data = metadata.tag_data
  22
  23 SCRIPTDIR = os.path.dirname(__file__)
  24
  25 CLASS_NAME = 'ToGo'
  26
  27 # Characters to remove from filenames
  28
  29 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  30            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  31
  32 # Default top-level share path
  33
  34 DEFPATH = '/TiVoConnect?Command=QueryContainer&Container=/NowPlaying'
  35
  36 # Some error/status message templates
  37
  38 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  39 "togo_path" before using this function.</p>"""
  40
  41 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  42 transfer to:</p> <p>%s</p>"""
  43
  44 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  45 <p>%s</p> <p>has been stopped.</p>"""
  46
  47 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  48 from the queue.</p>"""
  49
  50 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  51 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  52 incorrect Media Access Key. Please return to the Settings page and
  53 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  54
  55 # Preload the templates
  56 tnname = os.path.join(SCRIPTDIR, 'templates', 'npl.tmpl')
  57 NPL_TEMPLATE = file(tnname, 'rb').read()
  58
  59 mswindows = (sys.platform == "win32")
  60
  61 status = {} # Global variable to control download threads
  62 tivo_cache = {} # Cache of TiVo NPL
  63 queue = {} # Recordings to download -- list per TiVo
  64 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  65 details_urls = {} # URLs for extended data, indexed by main URL
  66
  67 def null_cookie(name, value):
  68     return cookielib.Cookie(0, name, value, None, False, '', False,
  69         False, '', False, False, None, False, None, None, None)
  70
  71 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  72 cj = cookielib.CookieJar()
  73 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  74 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  75                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  76                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  77
  78 tsn = config.get_server('togo_tsn')
  79 if tsn:
  80     tivo_opener.addheaders.append(('TSN', tsn))
  81
  82 class ToGo(Plugin):
  83     CONTENT_TYPE = 'text/html'
  84
  85     def tivo_open(self, url):
  86         # Loop just in case we get a server busy message
  87         while True:
  88             try:
  89                 # Open the URL using our authentication/cookie opener
  90                 return tivo_opener.open(url)
  91
  92             # Do a retry if the TiVo responds that the server is busy
  93             except urllib2.HTTPError, e:
  94                 if e.code == 503:
  95                     time.sleep(5)
  96                     continue
  97
  98                 # Log and throw the error otherwise
  99                 logger.error(e)
 100                 raise
 101
 102     def NPL(self, handler, query):
 103
 104         def getint(thing):
 105             try:
 106                 result = int(thing)
 107             except:
 108                 result = 0
 109             return result
 110
 111         global basic_meta
 112         global details_urls
 113         shows_per_page = 50 # Change this to alter the number of shows returned
 114         folder = ''
 115         FirstAnchor = ''
 116         has_tivodecode = bool(config.get_bin('tivodecode'))
 117
 118         if 'TiVo' in query:
 119             tivoIP = query['TiVo'][0]
 120             tsn = config.tivos_by_ip(tivoIP)
 121             attrs = config.tivos[tsn]
 122             tivo_name = attrs.get('name', tivoIP)
 123             tivo_mak = config.get_tsn('tivo_mak', tsn)
 124
 125             protocol = attrs.get('protocol', 'https')
 126             ip_port = '%s:%d' % (tivoIP, attrs.get('port', 443))
 127             path = attrs.get('path', DEFPATH)
 128             baseurl = '%s://%s%s' % (protocol, ip_port, path)
 129             theurl = baseurl
 130             if 'Folder' in query:
 131                 folder = query['Folder'][0]
 132                 theurl = urlparse.urljoin(theurl, folder)
 133             theurl += '&ItemCount=%d' % shows_per_page
 134             if 'AnchorItem' in query:
 135                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 136             if 'AnchorOffset' in query:
 137                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 138
 139             if (theurl not in tivo_cache or
 140                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 141                 # if page is not cached or old then retreive it
 142                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 143                 try:
 144                     page = self.tivo_open(theurl)
 145                 except IOError, e:
 146                     handler.redir(UNABLE % (tivoIP, cgi.escape(str(e))), 10)
 147                     return
 148                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 149                                       'thepage_time': time.time()}
 150                 page.close()
 151
 152             xmldoc = tivo_cache[theurl]['thepage']
 153             items = xmldoc.getElementsByTagName('Item')
 154             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 155             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 156             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 157             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 158             if items:
 159                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 160
 161             data = []
 162             for item in items:
 163                 entry = {}
 164                 for tag in ('CopyProtected', 'ContentType'):
 165                     value = tag_data(item, 'Details/' + tag)
 166                     if value:
 167                         entry[tag] = value
 168                 if entry['ContentType'].startswith('x-tivo-container'):
 169                     entry['Url'] = tag_data(item, 'Links/Content/Url')
 170                     entry['Title'] = tag_data(item, 'Details/Title')
 171                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 172                     lc = tag_data(item, 'Details/LastCaptureDate')
 173                     if not lc:
 174                         lc = tag_data(item, 'Details/LastChangeDate')
 175                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 176                         time.localtime(int(lc, 16)))
 177                 else:
 178                     keys = {'Icon': 'Links/CustomIcon/Url',
 179                             'Url': 'Links/Content/Url',
 180                             'Details': 'Links/TiVoVideoDetails/Url',
 181                             'SourceSize': 'Details/SourceSize',
 182                             'Duration': 'Details/Duration',
 183                             'CaptureDate': 'Details/CaptureDate'}
 184                     for key in keys:
 185                         value = tag_data(item, keys[key])
 186                         if value:
 187                             entry[key] = value
 188
 189                     if 'SourceSize' in entry:
 190                         rawsize = entry['SourceSize']
 191                         entry['SourceSize'] = metadata.human_size(rawsize)
 192
 193                     if 'Duration' in entry:
 194                         dur = getint(entry['Duration']) / 1000
 195                         entry['Duration'] = ( '%d:%02d:%02d' %
 196                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 197
 198                     if 'CaptureDate' in entry:
 199                         entry['CaptureDate'] = time.strftime('%b %d, %Y',
 200                             time.localtime(int(entry['CaptureDate'], 16)))
 201
 202                     url = urlparse.urljoin(baseurl, entry['Url'])
 203                     entry['Url'] = url
 204                     if url in basic_meta:
 205                         entry.update(basic_meta[url])
 206                     else:
 207                         basic_data = metadata.from_container(item)
 208                         entry.update(basic_data)
 209                         basic_meta[url] = basic_data
 210                         if 'Details' in entry:
 211                             details_urls[url] = entry['Details']
 212
 213                 data.append(entry)
 214         else:
 215             data = []
 216             tivoIP = ''
 217             TotalItems = 0
 218             ItemStart = 0
 219             ItemCount = 0
 220             title = ''
 221
 222         t = Template(NPL_TEMPLATE, filter=EncodeUnicode)
 223         t.quote = quote
 224         t.folder = folder
 225         t.status = status
 226         if tivoIP in queue:
 227             t.queue = queue[tivoIP]
 228         t.has_tivodecode = has_tivodecode
 229         t.togo_mpegts = config.is_ts_capable(tsn)
 230         t.tname = tivo_name
 231         t.tivoIP = tivoIP
 232         t.container = handler.cname
 233         t.data = data
 234         t.len = len
 235         t.TotalItems = getint(TotalItems)
 236         t.ItemStart = getint(ItemStart)
 237         t.ItemCount = getint(ItemCount)
 238         t.FirstAnchor = quote(FirstAnchor)
 239         t.shows_per_page = shows_per_page
 240         t.title = title
 241         handler.send_html(str(t), refresh='300')
 242
 243     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 244         # global status
 245         status[url].update({'running': True, 'queued': False})
 246
 247         parse_url = urlparse.urlparse(url)
 248
 249         name = unicode(unquote(parse_url[2]), 'utf-8').split('/')[-1].split('.')
 250         try:
 251             id = unquote(parse_url[4]).split('id=')[1]
 252             name.insert(-1, ' - ' + id)
 253         except:
 254             pass
 255         ts = status[url]['ts_format']
 256         if status[url]['decode']:
 257             if ts:
 258                 name[-1] = 'ts'
 259             else:
 260                 name[-1] = 'mpg'
 261         else:
 262             if ts:
 263                 name.insert(-1, ' (TS)')
 264             else:
 265                 name.insert(-1, ' (PS)')
 266         name.insert(-1, '.')
 267         name = ''.join(name)
 268         for ch in BADCHAR:
 269             name = name.replace(ch, BADCHAR[ch])
 270         outfile = os.path.join(togo_path, name)
 271
 272         if status[url]['save']:
 273             meta = basic_meta[url]
 274             try:
 275                 handle = self.tivo_open(details_urls[url])
 276                 meta.update(metadata.from_details(handle.read()))
 277                 handle.close()
 278             except:
 279                 pass
 280             metafile = open(outfile + '.txt', 'w')
 281             metadata.dump(metafile, meta)
 282             metafile.close()
 283
 284         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 285         try:
 286             if status[url]['ts_format']:
 287                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 288             else:
 289                 handle = self.tivo_open(url)
 290         except Exception, msg:
 291             status[url]['running'] = False
 292             status[url]['error'] = str(msg)
 293             return
 294
 295         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 296
 297         logger.info('[%s] Start getting "%s" from %s' %
 298                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 299
 300         if status[url]['decode']:
 301             fname = outfile
 302             if mswindows:
 303                 fname = fname.encode('cp1252')
 304             tivodecode_path = config.get_bin('tivodecode')
 305             tcmd = [tivodecode_path, '-m', mak, '-o', fname, '-']
 306             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 307                                           bufsize=(512 * 1024))
 308             f = tivodecode.stdin
 309         else:
 310             f = open(outfile, 'wb')
 311         length = 0
 312         start_time = time.time()
 313         last_interval = start_time
 314         now = start_time
 315         try:
 316             while status[url]['running']:
 317                 output = handle.read(1024000)
 318                 if not output:
 319                     break
 320                 length += len(output)
 321                 f.write(output)
 322                 now = time.time()
 323                 elapsed = now - last_interval
 324                 if elapsed >= 5:
 325                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 326                         (elapsed * 1024 * 1024))
 327                     status[url]['size'] += length
 328                     length = 0
 329                     last_interval = now
 330             if status[url]['running']:
 331                 status[url]['finished'] = True
 332         except Exception, msg:
 333             status[url]['running'] = False
 334             logger.info(msg)
 335         handle.close()
 336         f.close()
 337         status[url]['size'] += length
 338         if status[url]['running']:
 339             mega_elapsed = (now - start_time) * 1024 * 1024
 340             if mega_elapsed < 1:
 341                 mega_elapsed = 1
 342             size = status[url]['size']
 343             rate = size * 8.0 / mega_elapsed
 344             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 345                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 346                          tivo_name, size, rate))
 347             status[url]['running'] = False
 348         else:
 349             os.remove(outfile)
 350             if status[url]['save']:
 351                 os.remove(outfile + '.txt')
 352             logger.info('[%s] Transfer of "%s" from %s aborted' %
 353                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 354                          tivo_name))
 355             del status[url]
 356
 357     def process_queue(self, tivoIP, mak, togo_path):
 358         while queue[tivoIP]:
 359             time.sleep(5)
 360             url = queue[tivoIP][0]
 361             self.get_tivo_file(tivoIP, url, mak, togo_path)
 362             queue[tivoIP].pop(0)
 363         del queue[tivoIP]
 364
 365     def ToGo(self, handler, query):
 366         togo_path = config.get_server('togo_path')
 367         for name, data in config.getShares():
 368             if togo_path == name:
 369                 togo_path = data.get('path')
 370         if togo_path:
 371             tivoIP = query['TiVo'][0]
 372             tsn = config.tivos_by_ip(tivoIP)
 373             tivo_mak = config.get_tsn('tivo_mak', tsn)
 374             urls = query.get('Url', [])
 375             decode = 'decode' in query
 376             save = 'save' in query
 377             ts_format = 'ts_format' in query
 378             for theurl in urls:
 379                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 380                                   'queued': True, 'size': 0, 'finished': False,
 381                                   'decode': decode, 'save': save,
 382                                   'ts_format': ts_format}
 383                 if tivoIP in queue:
 384                     queue[tivoIP].append(theurl)
 385                 else:
 386                     queue[tivoIP] = [theurl]
 387                     thread.start_new_thread(ToGo.process_queue,
 388                                             (self, tivoIP, tivo_mak, togo_path))
 389                 logger.info('[%s] Queued "%s" for transfer to %s' %
 390                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 391                              unquote(theurl), togo_path))
 392             urlstring = '<br>'.join([unicode(unquote(x), 'utf-8')
 393                                      for x in urls])
 394             message = TRANS_QUEUE % (urlstring, togo_path)
 395         else:
 396             message = MISSING
 397         handler.redir(message, 5)
 398
 399     def ToGoStop(self, handler, query):
 400         theurl = query['Url'][0]
 401         status[theurl]['running'] = False
 402         handler.redir(TRANS_STOP % unquote(theurl))
 403
 404     def Unqueue(self, handler, query):
 405         theurl = query['Url'][0]
 406         tivoIP = query['TiVo'][0]
 407         del status[theurl]
 408         queue[tivoIP].remove(theurl)
 409         logger.info('[%s] Removed "%s" from queue' %
 410                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 411                      unquote(theurl)))
 412         handler.redir(UNQUEUE % unquote(theurl))