added unit test for feeds module
[straw.git] / src / lib / feeds.py
blobe4f1716057a20720a66d521b41fbcf5680f413b6
1 import locale, operator
2 import gobject
3 import ItemStore
4 import Config
6 class FeedList(gobject.GObject):
8 __gsignals__ = {
9 'changed' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()),
10 'updated' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE,
11 (gobject.TYPE_PYOBJECT, gobject.TYPE_PYOBJECT, gobject.TYPE_INT,)),
12 'imported' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE,
13 (gobject.TYPE_PYOBJECT, gobject.TYPE_PYOBJECT,
14 gobject.TYPE_BOOLEAN,)),
15 'deleted' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,))
18 def __init__(self, init_seq = []):
19 gobject.GObject.__init__(self)
20 self.feedlist = []
21 self._loading = False
23 def __iter__(self):
24 return iter(self.feedlist)
26 def load_data(self):
27 def _load(feedlist, parent):
28 for df in feedlist:
29 if isinstance(df, list):
30 _load(df[1:], parent)
31 else:
32 f = Feed.create_empty_feed()
33 f.undump(df)
34 self.append(parent, f)
35 self._loading = True
36 feedlist = Config.get_instance().feeds
37 if feedlist:
38 _load(feedlist, None)
39 self._loading = False
40 self.emit('changed')
42 def connect_signals(self, ob):
43 ob.connect('changed', self.feed_detail_changed)
45 # these signals are forwarded so that listeners who just want to
46 # listen for a specific event regardless of what feed it came from can
47 # just connect to this feedlist instead of connecting to the
48 # individual feeds.
49 #ob.signal_connect(Event.AllItemsReadSignal, self._forward_signal)
50 #ob.signal_connect(Event.ItemReadSignal, self._forward_signal)
51 #ob.signal_connect(Event.ItemsAddedSignal, self._forward_signal)
52 #ob.signal_connect(Event.FeedPolledSignal, self._forward_signal)
53 #ob.signal_connect(Event.FeedStatusChangedSignal, self._forward_signal)
54 #ob.signal_connect(Event.FeedErrorStatusChangedSignal, self._forward_signal)
56 def __setitem__(self, key, value):
57 self.feedlist.__setitem__(key, value)
58 self.connect_signals(value)
59 self.save_feeds_and_notify(True)
61 def extend(self, parent, values, from_sub=False):
62 list.extend(self.feedlist, values)
63 for f in values:
64 f.parent = parent
65 self.connect_signals(f)
66 self.save_feeds()
67 self.emit('imported', values, parent, from_sub)
69 def append(self, parent, value):
70 self.feedlist.append(value)
71 value.parent = parent
72 self.connect_signals(value)
73 self.save_feeds()
74 self.emit('updated', value, parent, -1)
76 def insert(self, index, parent, value):
77 self.feedlist.insert(index, value)
78 value.parent = parent
79 self.connect_signals(value)
80 self.save_feeds()
81 self.emit('updated', value, parent, index)
83 def index(self, feed):
84 return self.feedlist.index(feed)
86 def reorder(self, move, delta):
87 k = self.feedlist[:]
88 move = list(move)
89 move.sort()
90 if delta > 0:
91 move.reverse()
92 if move[0] == 0 and delta < 0 or move[-1] == (len(self.feedlist) - 1) and delta > 0:
93 return
94 for m in move:
95 k[m + delta], k[m] = k[m], k[m + delta]
96 for i in range(len(k)):
97 list.__setitem__(self.feedlist, i, k[i])
98 self.save_feeds()
99 self.emit('changed')
101 def __delitem__(self, value):
102 feed = self.feedlist[value]
103 list.__delitem__(self.feedlist, value)
104 feed.delete_all_items()
105 self.save_feeds()
106 self.emit('deleted', feed)
108 def save_feeds(self):
109 if not self._loading:
110 config = Config.get_instance()
111 config.feeds = [f.dump() for f in self.feedlist]
112 return
114 def feed_detail_changed(self, feed):
115 self.save_feeds()
116 self.emit('changed') # XXXX send the feed as well?
118 def _sort_dsu(self, seq):
119 aux_list = [(x.title, x) for x in seq]
120 aux_list.sort(lambda a,b:locale.strcoll(a[0],b[0]))
121 return [x[1] for x in aux_list]
123 def sort(self, indices = None):
124 if not indices or len(indices) == 1:
125 self[:] = self._sort_dsu(self)
126 else:
127 items = self._sort_dsu(indices)
128 for i,x in enumerate(items):
129 list.__setitem__(self, indices[i], items[i])
130 self.save_feeds()
131 self.emit('changed')
133 def __hash__(self):
134 h = 0
135 for item in self.feedlist:
136 h ^= hash(item)
137 return h
139 def get_feed_with_id(self, id):
140 for f in self.flatten_list():
141 if f.id == id:
142 return f
143 return None
145 def flatten_list(self, ob=None):
146 if ob is None:
147 ob = self.feedlist
148 l = []
149 for o in ob:
150 if isinstance(o, list):
151 l = l + self.flatten_list(o)
152 else:
153 l.append(o)
154 return l
156 feedlist_instance = None
157 def get_instance():
158 global feedlist_instance
159 if feedlist_instance is None:
160 feedlist_instance = FeedList()
161 return feedlist_instance
164 class IdleState(object):
165 ''' state for idle or normal operation '''
166 def __init__(self):
167 filename = os.path.join(utils.find_image_dir(), 'feed.png')
168 self.icon = gtk.gdk.pixbuf_new_from_file(filename)
170 @property
171 def icon(self):
172 return self.icon
174 @property
175 def mesg(self):
176 return None
178 class PollingState(object):
179 ''' state when feed is polling '''
180 def __init__(self):
181 self._icon = gtk.Image()
182 self._icon.set_from_stock(gtk.STOCK_EXECUTE, gtk.ICON_SIZE_MENU)
184 @property
185 def icon(self):
186 return self._icon.get_pixbuf()
188 @property
189 def mesg(self):
190 return None
192 class ErrorState(object):
193 ''' state when feed has errors '''
195 def __init__(self, mesg):
196 self._icon = gtk.Image()
197 self._icon.set_from_stock(gtk.STOCK_DIALOG_ERROR, gtk.ICON_SIZE_MENU)
198 self._mesg = mesg
200 @property
201 def icon(self):
202 return self._icon.get_pixbuf()
204 @property
205 def mesg(self):
206 return self._mesg
208 class FeedAdapter(gobject.GObject):
209 ''' Adapter to a feed entity
211 FeedAdapter wraps a feedparser.FeedParserDict object and delegates the
212 getter to get values from the wrapped object. For attributes that can be
213 set, it's best to create concrete attributes for that instead of mutating
214 the wrapped object. This mutable attributes can then be saved into the
215 on-disk representation.
217 The main point of this exercise is so that we can access feedparser data
218 structure's attributes instead of remapping it and readding stuff
219 everytime feedparser updates its API, or when we suddenly decided we need
220 some or all of feedparser's api.
222 FeedAdapter's responsibilities:
224 - core data comes from the wrapped object, FeedParserDict
225 - ability to dump and load itself at least using config persistence
226 - ability to manage its own items
227 - update items
228 - restore items
229 - ability to update the wrapped object
230 - ability to retain the number of items
231 - ability to set states (merge FeedDataRouter here):
232 - processing
233 - polling
234 - updating images, etc...
237 DEFAULT = -1
239 __save_fields = (('_title', ""), ('_location', ""), ('_username', ""),
240 ('_password', ""),('_id', ""), ('_error', None),
241 ('_items_stored', DEFAULT), ('_poll_freq', DEFAULT),
242 ('_last_poll', 0), #('_n_items_unread',0),
243 ('name',''),('description',''),
244 ('author',''),('copyright',''),
245 ('channel_creator', ""), # deprecate these soon
246 ('_channel_description', ""), ('_channel_title', ""),
247 ('_channel_link', ""), ('_channel_copyright', "")
250 __gsignals__ = {
251 'changed' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ())
254 def __init__(self, title, **kwargs):
255 gobject.GObject.__init__(self)
256 self._wrapped_data = kwargs.get('data', None)
257 self._title = title
258 self._location = kwargs.get('location',None)
259 self._username = kwargs.get('username',None)
260 self._password = kwargs.get('password',None)
261 self._state = kwargs.get('state',None)
262 self._id = FeedAdapter.DEFAULT
263 self._items_stored = FeedAdapter.DEFAULT
264 self._poll_freq = FeedAdapter.DEFAULT
265 self._last_poll = FeedAdapter.DEFAULT
266 self._items = FifoCache(num_entries=FeedAdapter.DEFAULT)
267 self._parent = None
268 self._error = None
270 def __str__(self):
271 return "Feed '%s' from %s" % (self._title, self._location)
273 def __getattr__(self, key):
274 try:
275 return getattr(self, key)
276 except AttributeError:
277 return getattr(self._wrapped, key)
279 def update(self, parserDict):
280 ''' updated the wrapped object '''
281 self._wrapped_data = parserDict
283 def setState(self, state):
284 ''' sets the state of self '''
285 self._state = state
287 @property
288 def state(self):
289 ''' gets the state of self '''
290 return self._state
292 @apply
293 def title():
294 doc = "The title of this Feed (as defined by user)"
295 def fget(self):
296 text = ''
297 if self._title:
298 text = self._title
299 return text
300 def fset(self, title):
301 if self._title != title:
302 self._title = title
303 return property(**locals())
305 @apply
306 def number_of_items_stored():
307 doc = 'the number of items to be stored for this feed'
308 def fget(self):
309 return self._items_stored
310 def fset(self, num=None):
311 if self._items_stored != num:
312 self._items_stored = num
313 return property(**locals())
315 @apply
316 def poll_frequency():
317 doc = 'the refresh frequency of this feed'
318 def fget(self):
319 return self._poll_freq
320 def fset(self, freq):
321 if self._poll_freq != freq:
322 self._poll_freq = freq
323 return property(**locals())
325 @apply
326 def last_poll():
327 doc = 'the time this feed was last updated'
328 def fget(self):
329 return self._last_poll
330 def fset(self, time):
331 if self._last_poll != time:
332 self._last_poll = time
333 return property(**locals())
335 @apply @deprecate
336 def error():
337 doc = 'error condition'
338 def fget(self):
339 print 'DEPRECATED: use feed.state.mesg instead'
340 return self._error
341 def fset(self, error):
342 print 'DEPRECATED: use feed.state.mesg instead'
343 if self._error != error:
344 self._error = error
345 return property(**locals())
347 @apply
348 def parent():
349 doc = 'the parent of this feed'
350 def fget(self):
351 return self._parent
352 def fset(self, parent):
353 self._parent = parent
354 return property(**locals())
356 def dump(self):
357 ''' dumps the attributes of this feed for storing '''
358 fl = {}
359 for f, default in self.__save_fields:
360 fl[f] = self.__getattribute__(f)
361 return fl
363 def undump(self, dict):
364 ''' sets the values of dict for this feed'''
365 for f, default in self.__save_fields:
366 self.__setattr__(f, dict.get(f, default))
367 return
370 class Feed(gobject.GObject):
371 "A Feed object stores information set by user about a RSS feed."
373 DEFAULT = -1
374 STATUS_IDLE = 0
375 STATUS_POLLING = 1
377 __slots__ = ('_title', '_location', '_username', '_password', '_parsed',
378 '__save_fields', '_items', '_slots',
379 '_id', '_channel_description',
380 '_channel_title', '_channel_link', '_channel_copyright',
381 'channel_lbd', 'channel_editor', 'channel_webmaster',
382 'channel_creator','_error', '_process_status', 'router', 'sticky', '_parent',
383 '_items_stored', '_poll_freq', '_last_poll','_n_items_unread')
385 __save_fields = (('_title', ""), ('_location', ""), ('_username', ""),
386 ('_password', ""), ('_id', ""),
387 ('_channel_description', ""), ('_channel_title', ""),
388 ('_channel_link', ""), ('_channel_copyright', ""),
389 ('channel_creator', ""), ('_error', None),
390 ('_items_stored', DEFAULT),
391 ('_poll_freq', DEFAULT),
392 ('_last_poll', 0),
393 ('_n_items_unread',0))
396 __gsignals__ = {
397 'changed' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()),
398 'poll-done' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()),
399 'items-added' :(gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE,
400 (gobject.TYPE_PYOBJECT,)),
401 'items-changed' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE,
402 (gobject.TYPE_PYOBJECT,)),
403 'items-deleted' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE,
404 (gobject.TYPE_PYOBJECT,))
408 # use one of the factory functions below instead of this directly
409 def __init__(self, title="", location="", username="", password=""):
410 import FeedDataRouter
411 gobject.GObject.__init__(self)
412 self._title = title
413 self._channel_description = ""
414 self._channel_title = ""
415 self._channel_link = ""
416 self._channel_copyright = ""
417 self.channel_lbd = None
418 self.channel_editor = ""
419 self.channel_webmaster = ""
420 self.channel_creator = ""
421 self._location = location
422 self._username = username
423 self._password = password
424 self._parsed = None
425 self._error = None
426 self._n_items_unread = 0
427 self._process_status = self.STATUS_IDLE
428 self.router = FeedDataRouter.FeedDataRouter(self)
429 self._parent = None
430 self._items_stored = Feed.DEFAULT
431 self._poll_freq = Feed.DEFAULT
432 self._last_poll = 0
434 self.config = Config.get_instance()
435 # XXX move this to subscriptions
436 self.config.connect('item-order-changed', self.item_order_changed_cb)
437 self.config.connect('item-stored-changed', self.item_stored_changed_cb)
438 self.item_order_reverse = self.config.item_order
439 self.item_stored_num = self.config.number_of_items_stored
440 self._items = FifoCache(num_entries=Feed.DEFAULT)
441 self._items_loaded = False
442 return
444 def __str__(self):
445 return "Feed '%s' from %s" % (self._title, self._location)
447 @property
448 def id(self):
449 return self._id
451 @apply
452 def parsed():
453 doc = "A ParsedSummary object generated from the summary file"
454 def fget(self):
455 return self._parsed
456 def fset(self, parsed):
457 self._parsed = parsed
458 return property(**locals())
460 @apply
461 def title():
462 doc = "The title of this Feed (as defined by user)"
463 def fget(self):
464 text = ''
465 if self._title:
466 text = self._title
467 return text
468 def fset(self, title):
469 if self._title != title:
470 self._title = title
471 self.emit('changed')
472 return property(**locals())
474 @apply
475 def access_info():
476 doc = "A tuple of location, username, password"
477 def fget(self):
478 return (self._location, self._username, self._password)
479 def fset(self, (location,username,password)):
480 self._location = location
481 self._username = username
482 self._password = password
483 self.emit('changed')
484 return property(**locals())
486 @apply
487 def location():
488 doc = ""
489 def fget(self):
490 return self._location
491 def fset(self, location):
492 if self._location != location:
493 self._location = location
494 self.emit('changed')
495 return property(**locals())
497 @apply
498 def channel_title():
499 doc = ""
500 def fget(self):
501 text = ''
502 if self._channel_title:
503 text = self._channel_title
504 return text
505 def fset(self, t):
506 changed = self._channel_title != t
507 self._channel_title = t
508 if changed:
509 self.emit('changed')
510 return property(**locals())
512 @apply
513 def channel_description():
514 doc = ""
515 def fget(self):
516 text = ''
517 if self._channel_description:
518 text = self._channel_description
519 return text
520 def fset(self, t):
521 changed = self._channel_description != t
522 self._channel_description = t
523 if changed:
524 self.emit('changed')
525 return property(**locals())
527 @apply
528 def channel_link():
529 doc = ""
530 def fget(self):
531 return self._channel_link
532 def fset(self, t):
533 changed = self._channel_link != t
534 self._channel_link = t
535 if changed:
536 self.emit('changed')
537 return property(**locals())
539 @apply
540 def channel_copyright():
541 doc = ""
542 def fget(self):
543 return self._channel_copyright
544 def fset(self, t):
545 changed = self._channel_copyright != t
546 self._channel_copyright = t
547 if changed:
548 self.emit('changed')
549 return property(**locals())
551 @apply
552 def number_of_items_stored():
553 doc = ""
554 def fget(self):
555 return self._items_stored
556 def fset(self, num=None):
557 if self._items_stored != num:
558 self._items_stored = num
559 return property(**locals())
561 @apply
562 def poll_frequency():
563 doc = ""
564 def fget(self):
565 return self._poll_freq
566 def fset(self, freq):
567 if self._poll_freq != freq:
568 self._poll_freq = freq
569 return property(**locals())
571 @apply
572 def last_poll():
573 doc = ""
574 def fget(self):
575 return self._last_poll
576 def fset(self, time):
577 if self._last_poll != time:
578 self._last_poll = time
579 return property(**locals())
581 @apply
582 def number_of_unread():
583 doc = "the number of unread items for this feed"
584 def fget(self):
585 return self._n_items_unread
586 def fset(self, n):
587 self._n_items_unread = n
588 self.emit('changed')
589 return property(**locals())
591 @apply
592 def error():
593 doc = ""
594 def fget(self):
595 return self._error
596 def fset(self, error):
597 if self._error != error:
598 self._error = error
599 self.emit('changed')
600 return property(**locals())
602 @apply
603 def process_status():
604 doc = ""
605 def fget(self):
606 return self._process_status
607 def fset(self, status):
608 if status != self._process_status:
609 self._process_status = status
610 self.emit('changed')
611 return property(**locals())
613 @apply
614 def parent():
615 doc = ""
616 def fget(self):
617 return self._parent
618 def fset(self, parent):
619 self._parent = parent
620 return property(**locals())
622 @property
623 def next_refresh(self):
624 """ return the feed's next refresh (time)"""
625 nr = None
626 if self._poll_freq == self.DEFAULT:
627 increment = self.config.poll_frequency
628 else:
629 increment = self._poll_freq
630 if increment > 0:
631 nr = self.last_poll + increment
632 return nr
634 def dump(self):
635 fl = {}
636 for f, default in self.__save_fields:
637 fl[f] = self.__getattribute__(f)
638 return fl
640 def undump(self, dict):
641 for f, default in self.__save_fields:
642 self.__setattr__(f, dict.get(f, default))
643 return
645 def poll_done(self):
646 self.emit('poll-done')
648 def item_order_changed_cb(self, config):
649 self.item_order_reverse = config.item_order
651 def item_stored_changed_cb(self, config):
652 self.item_stored_num = config.number_of_items_stored
654 def get_cutpoint(self):
655 cutpoint = self.number_of_items_stored
656 if cutpoint == Feed.DEFAULT:
657 cutpoint = self.item_stored_num
658 return cutpoint
661 def add_items(self, items):
662 if not self._items_loaded: self.load_contents()
663 items = sorted(items, key=operator.attrgetter('pub_date'),
664 reverse=self.item_order_reverse)
665 self._items.set_number_of_entries(self.get_cutpoint())
666 maxid = 0
667 if self._items:
668 maxid = reduce(max, [item.id for item in self._items.itervalues()])
669 newitems = filter(lambda x: x not in self._items.itervalues(), items)
670 for item in newitems:
671 maxid += 1
672 item.id = maxid
673 item.feed = self
674 item.connect('changed', self.item_changed_cb)
675 self._items[item.id] = item ## XXX pruned items don't get clean up properly
676 self.number_of_unread = len(filter(lambda x: not x.seen, self._items.itervalues()))
677 self.emit('items-added', newitems)
679 def restore_items(self, items):
680 items = sorted(items, key=operator.attrgetter('pub_date'),
681 reverse=self.item_order_reverse)
682 cutpoint = self.get_cutpoint()
683 self._items.set_number_of_entries(cutpoint)
684 olditems = []
685 for idx, item in enumerate(items):
686 if not item.sticky and idx >= cutpoint:
687 item.clean_up()
688 olditems.append(item)
689 continue
690 item.feed = self
691 item.connect('changed', self.item_changed_cb)
692 self._items[item.id] = item
693 self.number_of_unread = len(filter(lambda x: not x.seen, self._items.itervalues()))
694 if olditems: self.emit('items-deleted', olditems)
695 return
697 def item_changed_cb(self, item):
698 self.number_of_unread = len(filter(lambda x: not x.seen, self._items.itervalues()))
699 self.emit('items-changed', [item])
701 def delete_all_items(self):
702 self._items.clear()
703 self.emit('items-deleted', self._items.values())
705 @property
706 def items(self):
707 if not self._items_loaded:
708 self.load_contents()
709 return self._items.values()
711 def mark_items_as_read(self, items=None):
712 def mark(item): item.seen = True
713 unread = [item for item in self._items.itervalues() if not item.seen]
714 map(mark, unread)
715 self.emit('items-changed', unread)
717 def load_contents(self):
718 if self._items_loaded:
719 return False
720 itemstore = ItemStore.get_instance()
721 items = itemstore.read_feed_items(self)
722 print "feed.load_contents->items: ", len(items)
723 if items:
724 self.restore_items(items)
725 self._items_loaded = True
726 return self._items_loaded
728 def unload_contents(self):
729 if not self._items_loaded:
730 return
731 self._items.clear()
732 self._items_loaded = False
734 @classmethod
735 def create_new_feed(klass, title, location="", username="", password=""):
736 f = klass()
737 f._title = title
738 f._location = location
739 f._id = Config.get_instance().next_feed_id_seq()
740 f._username = username
741 f._password = password
742 return f
744 @classmethod
745 def create_empty_feed(klass):
746 f = klass()
747 return f
750 import UserDict
751 from collections import deque
753 class FifoCache(object, UserDict.DictMixin):
754 ''' A mapping that remembers the last 'num_entries' items that were set '''
756 def __init__(self, num_entries, dct=()):
757 self.num_entries = num_entries
758 self.dct = dict(dct)
759 self.lst = deque()
761 def __repr__(self):
762 return '%r(%r,%r)' % (
763 self.__class__.__name__, self.num_entries, self.dct)
765 def copy(self):
766 return self.__class__(self.num_entries, self.dct)
768 def keys(self):
769 return list(self.lst)
771 def __getitem__(self, key):
772 return self.dct[key]
774 def __setitem__(self, key, value):
775 dct = self.dct
776 lst = self.lst
777 if key in dct:
778 self.remove_from_deque(lst, key)
779 dct[key] = value
780 lst.append(key)
781 if len(lst) > self.num_entries:
782 del dct[lst.popleft()]
784 def __delitem__(self, key):
785 self.dct.pop(key)
786 self.remove_from_deque(self.lst, key)
788 # a method explicitly defined only as an optimization
789 def __contains__(self, item):
790 return item in self.dct
792 has_key = __contains__
794 def remove_from_deque(self, d, x):
795 for i, v in enumerate(d):
796 if v == x:
797 del d[i]
798 return
799 raise ValueError, '%r not in %r' % (x,d)
801 def set_number_of_entries(self, num):
802 self.num_entries = num
803 while len(self.lst) > num:
804 del self.dct[self.lst.popleft()]