plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import thread
   6 import time
   7 import urllib2
   8 import urlparse
   9 from urllib import quote, unquote
  10 from xml.dom import minidom
  11 from xml.sax.saxutils import escape
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Default top-level share path
  32
  33 DEFPATH = '/TiVoConnect?Command=QueryContainer&Container=/NowPlaying'
  34
  35 # Some error/status message templates
  36
  37 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  38 "togo_path" before using this function.</p>"""
  39
  40 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  41 transfer to:</p> <p>%s</p>"""
  42
  43 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  44 <p>%s</p> <p>has been stopped.</p>"""
  45
  46 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  47 from the queue.</p>"""
  48
  49 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  50 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  51 incorrect Media Access Key. Please return to the Settings page and
  52 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  53
  54 # Preload the templates
  55 tnname = os.path.join(SCRIPTDIR, 'templates', 'npl.tmpl')
  56 NPL_TEMPLATE = file(tnname, 'rb').read()
  57
  58 status = {} # Global variable to control download threads
  59 tivo_cache = {} # Cache of TiVo NPL
  60 queue = {} # Recordings to download -- list per TiVo
  61 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  62 details_urls = {} # URLs for extended data, indexed by main URL
  63
  64 def null_cookie(name, value):
  65     return cookielib.Cookie(0, name, value, None, False, '', False,
  66         False, '', False, False, None, False, None, None, None)
  67
  68 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  69 cj = cookielib.CookieJar()
  70 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  71 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  72                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  73                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  74
  75 class ToGo(Plugin):
  76     CONTENT_TYPE = 'text/html'
  77
  78     def tivo_open(self, url):
  79         # Loop just in case we get a server busy message
  80         while True:
  81             try:
  82                 # Open the URL using our authentication/cookie opener
  83                 return tivo_opener.open(url)
  84
  85             # Do a retry if the TiVo responds that the server is busy
  86             except urllib2.HTTPError, e:
  87                 if e.code == 503:
  88                     time.sleep(5)
  89                     continue
  90
  91                 # Log and throw the error otherwise
  92                 logger.error(e)
  93                 raise
  94
  95     def NPL(self, handler, query):
  96
  97         def getint(thing):
  98             try:
  99                 result = int(thing)
 100             except:
 101                 result = 0
 102             return result
 103
 104         global basic_meta
 105         global details_urls
 106         shows_per_page = 50 # Change this to alter the number of shows returned
 107         folder = ''
 108         FirstAnchor = ''
 109         has_tivodecode = bool(config.get_bin('tivodecode'))
 110
 111         if 'TiVo' in query:
 112             tivoIP = query['TiVo'][0]
 113             tsn = config.tivos_by_ip(tivoIP)
 114             attrs = config.tivos[tsn]
 115             tivo_name = attrs.get('name', tivoIP)
 116             tivo_mak = config.get_tsn('tivo_mak', tsn)
 117
 118             protocol = attrs.get('protocol', 'https')
 119             ip_port = '%s:%d' % (tivoIP, attrs.get('port', 443))
 120             path = attrs.get('path', DEFPATH)
 121             theurl = '%s://%s%s' % (protocol, ip_port, path)
 122             if 'Folder' in query:
 123                 folder = query['Folder'][0]
 124                 theurl = urlparse.urljoin(theurl, folder)
 125             theurl += '&ItemCount=%d' % shows_per_page
 126             if 'AnchorItem' in query:
 127                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 128             if 'AnchorOffset' in query:
 129                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 130
 131             if (theurl not in tivo_cache or
 132                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 133                 # if page is not cached or old then retreive it
 134                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 135                 try:
 136                     page = self.tivo_open(theurl)
 137                 except IOError, e:
 138                     handler.redir(UNABLE % (tivoIP, e), 10)
 139                     return
 140                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 141                                       'thepage_time': time.time()}
 142                 page.close()
 143
 144             xmldoc = tivo_cache[theurl]['thepage']
 145             items = xmldoc.getElementsByTagName('Item')
 146             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 147             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 148             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 149             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 150             if items:
 151                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 152
 153             data = []
 154             for item in items:
 155                 entry = {}
 156                 for tag in ('CopyProtected', 'ContentType'):
 157                     value = tag_data(item, 'Details/' + tag)
 158                     if value:
 159                         entry[tag] = value
 160                 if entry['ContentType'].startswith('x-tivo-container'):
 161                     entry['Url'] = tag_data(item, 'Links/Content/Url')
 162                     entry['Title'] = tag_data(item, 'Details/Title')
 163                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 164                     lc = tag_data(item, 'Details/LastCaptureDate')
 165                     if not lc:
 166                         lc = tag_data(item, 'Details/LastChangeDate')
 167                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 168                         time.localtime(int(lc, 16)))
 169                 else:
 170                     keys = {'Icon': 'Links/CustomIcon/Url',
 171                             'Url': 'Links/Content/Url',
 172                             'Details': 'Links/TiVoVideoDetails/Url',
 173                             'SourceSize': 'Details/SourceSize',
 174                             'Duration': 'Details/Duration',
 175                             'CaptureDate': 'Details/CaptureDate'}
 176                     for key in keys:
 177                         value = tag_data(item, keys[key])
 178                         if value:
 179                             entry[key] = value
 180
 181                     if 'SourceSize' in entry:
 182                         rawsize = entry['SourceSize']
 183                         entry['SourceSize'] = metadata.human_size(rawsize)
 184
 185                     if 'Duration' in entry:
 186                         dur = getint(entry['Duration']) / 1000
 187                         entry['Duration'] = ( '%d:%02d:%02d' %
 188                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 189
 190                     if 'CaptureDate' in entry:
 191                         entry['CaptureDate'] = time.strftime('%b %d, %Y',
 192                             time.localtime(int(entry['CaptureDate'], 16)))
 193
 194                     url = entry['Url']
 195                     if url in basic_meta:
 196                         entry.update(basic_meta[url])
 197                     else:
 198                         basic_data = metadata.from_container(item)
 199                         entry.update(basic_data)
 200                         basic_meta[url] = basic_data
 201                         details_urls[url] = entry['Details']
 202
 203                 data.append(entry)
 204         else:
 205             data = []
 206             tivoIP = ''
 207             TotalItems = 0
 208             ItemStart = 0
 209             ItemCount = 0
 210             title = ''
 211
 212         t = Template(NPL_TEMPLATE, filter=EncodeUnicode)
 213         t.escape = escape
 214         t.quote = quote
 215         t.folder = folder
 216         t.status = status
 217         if tivoIP in queue:
 218             t.queue = queue[tivoIP]
 219         t.has_tivodecode = has_tivodecode
 220         t.togo_mpegts = config.is_ts_capable(tsn)
 221         t.tname = tivo_name
 222         t.tivoIP = tivoIP
 223         t.container = handler.cname
 224         t.data = data
 225         t.len = len
 226         t.TotalItems = getint(TotalItems)
 227         t.ItemStart = getint(ItemStart)
 228         t.ItemCount = getint(ItemCount)
 229         t.FirstAnchor = quote(FirstAnchor)
 230         t.shows_per_page = shows_per_page
 231         t.title = title
 232         handler.send_html(str(t), refresh='300')
 233
 234     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 235         # global status
 236         status[url].update({'running': True, 'queued': False})
 237
 238         parse_url = urlparse.urlparse(url)
 239
 240         name = unquote(parse_url[2])[10:].split('.')
 241         try:
 242             id = unquote(parse_url[4]).split('id=')[1]
 243             name.insert(-1, ' - ' + id)
 244         except:
 245             pass
 246         ts = status[url]['ts_format']
 247         if status[url]['decode']:
 248             if ts:
 249                 name[-1] = 'ts'
 250             else:
 251                 name[-1] = 'mpg'
 252         else:
 253             if ts:
 254                 name.insert(-1, ' (TS)')
 255             else:
 256                 name.insert(-1, ' (PS)')
 257         name.insert(-1, '.')
 258         name = ''.join(name)
 259         for ch in BADCHAR:
 260             name = name.replace(ch, BADCHAR[ch])
 261         outfile = os.path.join(togo_path, name)
 262
 263         if status[url]['save']:
 264             meta = basic_meta[url]
 265             try:
 266                 handle = self.tivo_open(details_urls[url])
 267                 meta.update(metadata.from_details(handle.read()))
 268                 handle.close()
 269             except:
 270                 pass
 271             metafile = open(outfile + '.txt', 'w')
 272             metadata.dump(metafile, meta)
 273             metafile.close()
 274
 275         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 276         try:
 277             if status[url]['ts_format']:
 278                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 279             else:
 280                 handle = self.tivo_open(url)
 281         except Exception, msg:
 282             status[url]['running'] = False
 283             status[url]['error'] = str(msg)
 284             return
 285
 286         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 287
 288         logger.info('[%s] Start getting "%s" from %s' %
 289                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 290
 291         if status[url]['decode']:
 292             tivodecode_path = config.get_bin('tivodecode')
 293             tcmd = [tivodecode_path, '-m', mak, '-o', outfile, '-']
 294             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 295                                           bufsize=(512 * 1024))
 296             f = tivodecode.stdin
 297         else:
 298             f = open(outfile, 'wb')
 299         length = 0
 300         start_time = time.time()
 301         last_interval = start_time
 302         now = start_time
 303         try:
 304             while status[url]['running']:
 305                 output = handle.read(1024000)
 306                 if not output:
 307                     break
 308                 length += len(output)
 309                 f.write(output)
 310                 now = time.time()
 311                 elapsed = now - last_interval
 312                 if elapsed >= 5:
 313                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 314                         (elapsed * 1024 * 1024))
 315                     status[url]['size'] += length
 316                     length = 0
 317                     last_interval = now
 318             if status[url]['running']:
 319                 status[url]['finished'] = True
 320         except Exception, msg:
 321             status[url]['running'] = False
 322             logger.info(msg)
 323         handle.close()
 324         f.close()
 325         status[url]['size'] += length
 326         if status[url]['running']:
 327             mega_elapsed = (now - start_time) * 1024 * 1024
 328             if mega_elapsed < 1:
 329                 mega_elapsed = 1
 330             size = status[url]['size']
 331             rate = size * 8.0 / mega_elapsed
 332             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 333                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 334                          tivo_name, size, rate))
 335             status[url]['running'] = False
 336         else:
 337             os.remove(outfile)
 338             if status[url]['save']:
 339                 os.remove(outfile + '.txt')
 340             logger.info('[%s] Transfer of "%s" from %s aborted' %
 341                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 342                          tivo_name))
 343             del status[url]
 344
 345     def process_queue(self, tivoIP, mak, togo_path):
 346         while queue[tivoIP]:
 347             time.sleep(5)
 348             url = queue[tivoIP][0]
 349             self.get_tivo_file(tivoIP, url, mak, togo_path)
 350             queue[tivoIP].pop(0)
 351         del queue[tivoIP]
 352
 353     def ToGo(self, handler, query):
 354         togo_path = config.get_server('togo_path')
 355         for name, data in config.getShares():
 356             if togo_path == name:
 357                 togo_path = data.get('path')
 358         if togo_path:
 359             tivoIP = query['TiVo'][0]
 360             tsn = config.tivos_by_ip(tivoIP)
 361             tivo_mak = config.get_tsn('tivo_mak', tsn)
 362             urls = query.get('Url', [])
 363             decode = 'decode' in query
 364             save = 'save' in query
 365             ts_format = 'ts_format' in query
 366             for theurl in urls:
 367                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 368                                   'queued': True, 'size': 0, 'finished': False,
 369                                   'decode': decode, 'save': save,
 370                                   'ts_format': ts_format}
 371                 if tivoIP in queue:
 372                     queue[tivoIP].append(theurl)
 373                 else:
 374                     queue[tivoIP] = [theurl]
 375                     thread.start_new_thread(ToGo.process_queue,
 376                                             (self, tivoIP, tivo_mak, togo_path))
 377                 logger.info('[%s] Queued "%s" for transfer to %s' %
 378                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 379                              unquote(theurl), togo_path))
 380             urlstring = '<br>'.join([unquote(x) for x in urls])
 381             message = TRANS_QUEUE % (urlstring, togo_path)
 382         else:
 383             message = MISSING
 384         handler.redir(message, 5)
 385
 386     def ToGoStop(self, handler, query):
 387         theurl = query['Url'][0]
 388         status[theurl]['running'] = False
 389         handler.redir(TRANS_STOP % unquote(theurl))
 390
 391     def Unqueue(self, handler, query):
 392         theurl = query['Url'][0]
 393         tivoIP = query['TiVo'][0]
 394         del status[theurl]
 395         queue[tivoIP].remove(theurl)
 396         logger.info('[%s] Removed "%s" from queue' %
 397                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 398                      unquote(theurl)))
 399         handler.redir(UNQUEUE % unquote(theurl))