straw/FeedUpdater.py

   1 """ FeedUpdater.py
   2
   3 """
   4
   5 __copyright__ = "Copyright (c) 2002-2005 Free Software Foundation, Inc."
   6 __license__ = """
   7 Straw is free software; you can redistribute it and/or modify it under the
   8 terms of the GNU General Public License as published by the Free Software
   9 Foundation; either version 2 of the License, or (at your option) any later
  10 version.
  11
  12 Straw is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  14 A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  15
  16 You should have received a copy of the GNU General Public License along with
  17 this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  18 Place - Suite 330, Boston, MA 02111-1307, USA. """
  19
  20 from straw.JobManager import Job, TaskResult, TaskThread, TaskInfo, ThreadPoolJobHandler
  21 import time
  22 from threading import Lock
  23
  24 class FeedUpdateJobHandler(ThreadPoolJobHandler):
  25     job_id = "feed-update"
  26
  27     def __init__(self, job):
  28         ThreadPoolJobHandler.__init__(self, job)
  29
  30         self.pool_size = 3
  31         self.result_class = FeedUpdateTaskResult
  32         self.task_class = FeedUpdateTaskThread
  33
  34     def _split(self):
  35         i = 0
  36         for a in self.job.data:
  37             i += 1
  38             ti = TaskInfo(i, { "feed" : a })
  39             self.task_queue.put(ti)
  40
  41     def _prepare_result(self):
  42         list = []
  43
  44         while not self.result_queue.empty():
  45             list.append(self.result_queue.get())
  46
  47         return list
  48
  49 class FeedUpdateTaskResult(TaskResult):
  50     def __init__(self, task_info, result):
  51         self.task_info = task_info
  52         self.result = result
  53
  54     def get(self):
  55         raise NotImplementedError
  56
  57 class FeedUpdateTaskThread(TaskThread):
  58     def __init__(self, handler, task_queue, result_queue):
  59         TaskThread.__init__(self, handler, task_queue, result_queue)
  60
  61     def _process(self, task):
  62         import httplib2
  63         feed = task.data["feed"]
  64
  65         import os
  66         from straw import Config
  67         CACHE_DIR = os.path.join(Config.straw_home(), 'cache')
  68         h = httplib2.Http(CACHE_DIR)
  69         #httplib2.debuglevel=4
  70         url = feed.location#feed.location
  71         resp, content = h.request(url, "GET")
  72         parsed = None
  73
  74         print resp.status
  75
  76         if not resp.fromcache:
  77             import SummaryParser
  78             parsed = SummaryParser.parse(content, feed)
  79             i = 0
  80             for image in sum([item.images for item in parsed.items], []):
  81                 resp, content = h.request(image, "GET")
  82                 i = i + 1
  83                 #f = open("/home/ppawel/Desktop/test/%s" % i, "w")
  84                 #f.write(content)
  85                 #f.close()
  86         else:
  87             print "304: %s" % url
  88
  89         return parsed
  90
  91     def _prepare_task_info(self, task_info):
  92         return task_info.data["feed"]
  93
  94 import straw.JobManager as JobManager
  95
  96 JobManager.register_handler(FeedUpdateJobHandler)
  97
  98 def update_urls(urls, observers):
  99     update = Job("feed-update")
 100     update.data = urls
 101     update.observers = observers
 102     JobManager.start_job(update, True)
 103
 104 def update_feeds(feeds, observers):
 105     update = Job("feed-update")
 106     update.data = feeds
 107     update.observers = observers
 108     JobManager.start_job(update)