straw/FeedUpdater.py

   1 """ FeedUpdater.py
   2
   3 """
   4
   5 __copyright__ = "Copyright (c) 2002-2005 Free Software Foundation, Inc."
   6 __license__ = """
   7 Straw is free software; you can redistribute it and/or modify it under the
   8 terms of the GNU General Public License as published by the Free Software
   9 Foundation; either version 2 of the License, or (at your option) any later
  10 version.
  11
  12 Straw is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  14 A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  15
  16 You should have received a copy of the GNU General Public License along with
  17 this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  18 Place - Suite 330, Boston, MA 02111-1307, USA. """
  19
  20 from straw.JobManager import Job, TaskResult, TaskThread, TaskInfo, ThreadPoolJobHandler
  21 import time
  22 from threading import Lock
  23
  24 class FeedUpdateJobHandler(ThreadPoolJobHandler):
  25     job_id = "feed-update"
  26
  27     def __init__(self, job):
  28         ThreadPoolJobHandler.__init__(self, job)
  29
  30         self.pool_size = 3
  31         self.result_class = FeedUpdateTaskResult
  32         self.task_class = FeedUpdateTaskThread
  33
  34     def _split(self):
  35         i = 0
  36         for a in self.job.data:
  37             i += 1
  38             ti = TaskInfo(i, { "feed" : a })
  39             self.task_queue.put(ti)
  40
  41     def _prepare_result(self):
  42         list = []
  43
  44         while not self.result_queue.empty():
  45             list.append(self.result_queue.get())
  46
  47         return list
  48
  49 class FeedUpdateTaskResult(TaskResult):
  50     def __init__(self, task_info, result):
  51         self.task_info = task_info
  52         self.result = result
  53
  54     def get(self):
  55         raise NotImplementedError
  56
  57 class FeedUpdateTaskThread(TaskThread):
  58     def __init__(self, handler, task_queue, result_queue):
  59         TaskThread.__init__(self, handler, task_queue, result_queue)
  60
  61     def _process(self, task):
  62         import httplib2
  63         feed = task.data["feed"]
  64
  65         import os
  66         from straw import Config
  67         CACHE_DIR = os.path.join(Config.straw_home(), 'cache')
  68         h = httplib2.Http(CACHE_DIR)
  69         #httplib2.debuglevel=4
  70         url = feed.location#feed.location
  71         resp, content = h.request(url, "GET")
  72         parsed = None
  73
  74         print resp.fromcache
  75
  76         import SummaryParser
  77         parsed = SummaryParser.parse(content, feed)
  78
  79         #i = 0
  80         #for image in sum([item.images for item in parsed.items], []):
  81             #resp, content = h.request(image, "GET")
  82             #    i = i + 1
  83                 #f = open("/home/ppawel/Desktop/test/%s" % i, "w")
  84                 #f.write(content)
  85                 #f.close()
  86
  87         return parsed
  88
  89     def _prepare_task_info(self, task_info):
  90         return task_info.data["feed"]
  91
  92 import straw.JobManager as JobManager
  93
  94 JobManager.register_handler(FeedUpdateJobHandler)
  95
  96 def update_urls(urls, observers):
  97     update = Job("feed-update")
  98     update.data = urls
  99     update.observers = observers
 100     JobManager.start_job(update, True)
 101
 102 def update_feeds(feeds, observers):
 103     update = Job("feed-update")
 104     update.data = feeds
 105     update.observers = observers
 106     JobManager.start_job(update)