catcher

   1 #!/usr/bin/env python
   2
   3 from __future__ import with_statement
   4
   5 import os, os.path
   6 import pickle
   7 from subprocess import Popen
   8 import sys
   9 import conf
  10
  11 import logging
  12
  13 os.environ['DJANGO_SETTINGS_MODULE'] = 'riffle.settings'
  14 from riffle.catcher import models as catcher
  15
  16 # logging setup
  17 logging.basicConfig(
  18         level=logging.DEBUG,
  19         format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
  20         datefmt='%m-%d %H:%M',
  21         filename='catcher.log',
  22         filemode='w')
  23 console = logging.StreamHandler()
  24 console.setLevel(logging.DEBUG if '-v' in sys.argv else logging.INFO)
  25 formatter = logging.Formatter('%(levelname)-8s %(message)s')
  26 console.setFormatter(formatter)
  27 logging.getLogger('').addHandler(console)
  28
  29 def try_load(fname, default):
  30     try:
  31         with open(fname,"r") as f:
  32             return pickle.load(f)
  33     except IOError:
  34         return default
  35
  36 url_files = try_load("files.db", {})
  37
  38 for feed in catcher.Feed.objects.all():
  39     feed.refresh(url_files)
  40
  41 exit(0)
  42
  43 def make_local_path(url):
  44     if url in url_files: return url_files[url]
  45     desired = os.path.join( conf.media_dir, url_basename(url) )
  46     desired_r,desired_e = os.path.splitext(desired)
  47     attempt = 0
  48     while desired in files:
  49         attempt += 1
  50         desired = desired_r + str(attempt) + desired_e
  51     else:
  52         url_files[url] = desired
  53         files.add( desired )
  54         save(url_files, "files.db")
  55         return desired
  56
  57 def make_tmp_path(path):
  58     return path + ".part"
  59
  60 def wget(url, fname):
  61     dir = os.path.dirname(fname)
  62     if not os.path.isdir(dir):
  63         os.makedirs(dir)
  64     cmd = "wget -c '%s' -O %s" % (url, fname)
  65     logging.info("Exec: %s", cmd)
  66     if "-n" in sys.argv:
  67         return False
  68     else:
  69         return Popen(cmd, shell=True).wait() == 0
  70
  71 def download_episode(url):
  72     local_path = make_local_path(url)
  73     tmp_file = make_tmp_path(local_path)
  74     if wget(url, tmp_file):
  75         logging.info("Renaming %s to %s", tmp_file, local_path)
  76         os.rename(tmp_file, local_path)
  77         return True
  78     return False
  79
  80 dl_queue = try_load("dl-queue.db", [])
  81
  82 for feed in get_updated_feeds():
  83     logging.info("Checking feed: %s", feed)
  84     # FIXME for now limit to 10 but
  85     # (a) have to make sure it's sorted by age
  86     # (b) should be configurable and overridable
  87     i=10
  88     for url in feed.get_new_episodes():
  89         if url not in url_files and url not in dl_queue:
  90             logging.info("Queueing for download: %s", url)
  91             dl_queue.append(url)
  92         else:
  93             logging.debug("Ignoring familiar url: %s (%s)", url, feed)
  94         i -= 1
  95         if i == 0:
  96             break
  97     feed.update_timestamp()
  98
  99 save(feeds, "feeds.db")
 100 save(dl_queue, "dl-queue.db")
 101
 102 # traverse a copy, we seem to miss episodes otherwise
 103 for url in [x for x in dl_queue]:
 104     if download_episode(url):
 105         dl_queue.remove(url)
 106         save(dl_queue, "dl-queue.db")
 107