From 812316cbdb922e3f1690594fb8f80d7f98f13335 Mon Sep 17 00:00:00 2001 From: DrFrasierCrane Date: Sun, 9 Dec 2007 13:56:01 +0100 Subject: [PATCH] Fixes (workarounds) in OPML parsing, more work on GUI... --- data/sql/create_01.sql | 12 ++++-- straw/Application.py | 26 ++++++------- straw/FeedListView.py | 66 ++++++++++++++++++++----------- straw/FeedManager.py | 88 ++++++++++++++++++++++++++++-------------- straw/FeedUpdater.py | 22 +++++------ straw/error.py | 5 +-- straw/opml.py | 38 ++++++++++-------- straw/storage/SQLiteDAO.py | 12 +++++- straw/storage/SQLiteStorage.py | 32 ++++++++++++--- straw/subscribe.py | 59 +++++++++++++++------------- 10 files changed, 224 insertions(+), 136 deletions(-) diff --git a/data/sql/create_01.sql b/data/sql/create_01.sql index bb8e87d..f58e489 100644 --- a/data/sql/create_01.sql +++ b/data/sql/create_01.sql @@ -27,10 +27,14 @@ CREATE TABLE IF NOT EXISTS items ( description TEXT NOT NULL ); -- -INSERT INTO nodes (id, parent_id, type, norder) VALUES (1, NULL, 'C', 0); +CREATE INDEX idx_nodes_id ON nodes (id) -- -INSERT INTO nodes (id, parent_id, type, norder) VALUES (2, 1, 'C', 0); +CREATE INDEX idx_feeds_id ON feeds (id) -- -INSERT INTO categories (id, name) VALUES (1, 'root'); +CREATE INDEX idx_categories_id ON categories (id) +-- +CREATE INDEX idx_items_feed_id ON items (feed_id) -- -INSERT INTO categories (id, name) VALUES (2, 'root2'); +INSERT INTO nodes (id, parent_id, type, norder) VALUES (1, NULL, 'C', 0); +-- +INSERT INTO categories (id, name) VALUES (1, 'root'); diff --git a/straw/Application.py b/straw/Application.py index 2adef92..b7c551e 100644 --- a/straw/Application.py +++ b/straw/Application.py @@ -254,7 +254,7 @@ class ApplicationPresenter(MVP.BasicPresenter): "task-done": [ self._on_feed_poll_done ] }) def _on_feed_poll_started(self, handler, feed): - print feed.id + pass#print feed.id def _on_feed_poll_done(self, handler, data): pass @@ -388,23 +388,22 @@ class ApplicationPresenter(MVP.BasicPresenter): gtk.RESPONSE_CANCEL, gtk.STOCK_OK, gtk.RESPONSE_OK)) category_list = [] + + for category in FeedManager.categories(): + category_list.append(category) + + print category_list + model = gtk.ListStore(gobject.TYPE_STRING, gobject.TYPE_PYOBJECT) combobox = gtk.ComboBox(model) celltitle = gtk.CellRendererText() combobox.pack_start(celltitle,False) combobox.add_attribute(celltitle, 'text', 0) - # add 'All' Category - it = model.append() - #model.set(it, 0, category_list.all_category.title, - # 1, category_list.all_category) + for category in category_list: + it = model.append() + model.set(it, 0, category.name, 1, category) - # add user categories - #for category in category_list.user_categories: - # it = model.append() - # model.set(it, 0, category.title, 1, category) - - # .. combobox.set_active(0) label = gtk.Label(extra_widget_title) label.set_alignment(1.0,0.5) @@ -416,7 +415,6 @@ class ApplicationPresenter(MVP.BasicPresenter): dialog.set_extra_widget(hbox) return (dialog, combobox) - def import_subscriptions(self, parent): (dialog,combobox) = self._setup_filechooser_dialog(_("Import Subscriptions"), gtk.FILE_CHOOSER_ACTION_OPEN, @@ -431,9 +429,9 @@ class ApplicationPresenter(MVP.BasicPresenter): if response == gtk.RESPONSE_OK: filename = dialog.get_filename() model = combobox.get_model() - cat = model[combobox.get_active()][1] + category = model[combobox.get_active()][1] dialog.hide() - FeedManager.import_opml(filename) + FeedManager.import_opml(filename, category) dialog.destroy() def export_subscriptions(parent): diff --git a/straw/FeedListView.py b/straw/FeedListView.py index 20a7ba7..0390869 100644 --- a/straw/FeedListView.py +++ b/straw/FeedListView.py @@ -34,8 +34,10 @@ class Column: class TreeViewNode(object): - def __init__(self, node): + def __init__(self, node, store): self.node = node + self.store = store + self.setup_node() def setup_node(self): @@ -90,6 +92,35 @@ class TreeViewNode(object): logging.exception(ex) return widget.render_icon(gtk.STOCK_DIRECTORY, gtk.ICON_SIZE_MENU) return self.default_pixbuf + + @property + def parent_path(self): + import copy + + path = [] + + if self.node.parent != None: + node = copy.copy(self.node.parent) + + while node != None: + path.append(str(node.norder)) + node = copy.copy(node.parent) + + path.pop() + path.reverse() + if len(path) == 0: + return None + else: + return ":".join(path) + + @property + def parent_iter(self): + path = self.parent_path + + if path == None: + return None + else: + return self.store.get_iter_from_string(path) class FeedListModel: ''' The model for the feed list view ''' @@ -101,10 +132,10 @@ class FeedListModel: def refresh_tree(self): self.categories, self.appmodel = FeedManager.get_model() - self._prepare_model() self._prepare_store() + self._prepare_model() - print self.tv_nodes + #print self.tv_nodes[1] self._populate_tree(1, None, []) @@ -116,7 +147,7 @@ class FeedListModel: self.tv_nodes = {} for parent_id in self.appmodel.keys(): - self.tv_nodes[parent_id] = [TreeViewNode(node) for node in self.appmodel[parent_id]] + self.tv_nodes[parent_id] = [TreeViewNode(node, self.store) for node in self.appmodel[parent_id]] def _prepare_store(self): self.store = gtk.TreeStore(gtk.gdk.Pixbuf, str, str, str, gobject.TYPE_PYOBJECT) @@ -129,18 +160,18 @@ class FeedListModel: node = tv_node.node if node.type == "F": - tv_node.treeiter = self._create_row(tv_node, parent_iter) + tv_node.treeiter = self._create_row(tv_node) tv_node.store = self.store elif node.type == "C": - current_parent = self._create_row(tv_node, parent_iter) + current_parent = self._create_row(tv_node) tv_node.treeiter = current_parent tv_node.store = self.store if self.tv_nodes.has_key(node.id): self._populate_tree(node.id, current_parent, done) - def _create_row(self, node, parent = None): - return self.store.append(parent, [node.pixbuf, + def _create_row(self, node): + return self.store.append(node.parent_iter, [node.pixbuf, node.title, 'black', node.unread_count, @@ -158,26 +189,15 @@ class FeedListModel: self.tv_nodes[tv_node.node.parent_id].append(tv_node) def node_added_cb(self, src, node): - tv_node = TreeViewNode(node) + tv_node = TreeViewNode(node, self.store) self.add_node(tv_node) - parent_node = self._lookup_parent(tv_node) - parent_iter = None - - node = tv_node.node - s = [] - while node != None: - s.append(str(node.norder)) - node = node.parent - s.reverse() - s.pop() - #print ":".join(s) - parent_iter = self.store.get_iter_from_string(":".join(s)) + self._create_row(tv_node) - if parent_node != None and hasattr(parent_node, "treeiter"): + """if parent_node != None and hasattr(parent_node, "treeiter"): parent_iter = parent_node.treeiter tv_node.treeiter = self._create_row(tv_node, parent_iter) - tv_node.store = self.store + tv_node.store = self.store""" @property def model(self): diff --git a/straw/FeedManager.py b/straw/FeedManager.py index 0463687..5625508 100644 --- a/straw/FeedManager.py +++ b/straw/FeedManager.py @@ -11,9 +11,9 @@ _storage_path = None model_data = None -def import_opml(path): +def import_opml(path, category): fm = _get_instance() - fm.import_opml(path) + fm.import_opml(path, category) def init(): fm = _get_instance() @@ -56,10 +56,26 @@ def save_feed(feed): result = fm.save_feed(feed) return result +def save_all(save_list): + fm = _get_instance() + fm.save_all(save_list) + def get_model(): fm = _get_instance() return fm.categories, fm.model +def categories(root_id = 1): + _categories, model = get_model() + + yield _categories[root_id] + + if model.has_key(root_id): + for node in model[root_id]: + if node.type == "C": + for child in categories(node.id): + yield child + return + class FeedManager(GObject): __gsignals__ = { "item-added" : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,)), @@ -75,45 +91,49 @@ class FeedManager(GObject): def init_storage(self, path): self.storage = Storage(path) self.dao = DAO(self.storage) - - #self.categories = self.get_categories() self.nodes, self.model = self.dao.get_nodes() - - #self.feeds = dict([(node.id, node) for node in self.nodes.values() if node.type == "F"]) - #print self.model - self._process_model() - - def import_opml(self, path): + + def import_opml(self, path, category): job = Job("opml-parse") - job.data = path + job.data = (path, category) job.observers = [ { "job-done": [ self.import_opml_save ] } ] + JobManager.start_job(job) def import_opml_save(self, handler, opml_data): - categories = opml_data - #print opml_data - self._save_tree(opml_data) - #for item in opml_data: - # if isinstance(item, Feed): - # self.save_feed(item) - # else: - # self.save_category(item) - - def _save_tree(self, level, parent_id = 1): - for item in level: + save_list, category = opml_data + + self.dao.tx_begin() + self.save_all(save_list) + self.dao.tx_commit() + + def _model_add_node(self, node): + if not self.model.has_key(node.parent_id): + self.model[node.parent_id] = [] + + self.model[node.parent_id].append(node) + + def save_all(self, save_list): + for item in save_list: if isinstance(item, Feed): - parent_id = 1 + item.parent_id = 1 + if item.parent: - parent_id = item.parent.id - item.parent_id = parent_id + item.parent_id = item.parent.id + else: + item.parent_id = 1 + item.parent = self.nodes[1] + self.save_feed(item) else: - parent_id = 1 if item.parent: - parent_id = item.parent.id - item.parent_id = parent_id + item.parent_id = item.parent.id + else: + item.parent_id = 1 + item.parent = self.nodes[1] + self.save_category(item) def lookup_feed(self, id): @@ -139,6 +159,9 @@ class FeedManager(GObject): feed.connect("unread-count-changed", category.on_unread_count_changed) self.feeds[feed.parent_id].append(feed) + self.nodes[feed.id] = feed + self._model_add_node(feed) + self.emit("feed-added", feed) return result @@ -154,7 +177,7 @@ class FeedManager(GObject): def update_all_feeds_done(self, handler, data): pass - + def _on_update_feed_start(self, handler, feed): feed.props.status = straw.FS_UPDATING @@ -177,9 +200,14 @@ class FeedManager(GObject): def save_category(self, category): self.dao.save(category) + category.connect("unread-count-changed", self.lookup_category(category.parent_id).on_unread_count_changed) + self.categories[category.id] = category + self.nodes[category.id] = category + self._model_add_node(category) + self.emit("category-added", category) # ITEMS @@ -245,6 +273,8 @@ class FeedManager(GObject): for node in self.nodes.values(): if node.parent_id != None: node.parent = self.nodes[node.parent_id] + else: + node.parent = None if node.type == "F": if not self.feeds.has_key(node.parent_id): diff --git a/straw/FeedUpdater.py b/straw/FeedUpdater.py index 99bcfbd..0491c2f 100644 --- a/straw/FeedUpdater.py +++ b/straw/FeedUpdater.py @@ -70,21 +70,19 @@ class FeedUpdateTaskThread(TaskThread): url = feed.location#feed.location resp, content = h.request(url, "GET") parsed = None - - #print resp.status - - if not resp.fromcache: - import SummaryParser - parsed = SummaryParser.parse(content, feed) - i = 0 - for image in sum([item.images for item in parsed.items], []): - resp, content = h.request(image, "GET") - i = i + 1 + + print resp.fromcache + + import SummaryParser + parsed = SummaryParser.parse(content, feed) + + #i = 0 + #for image in sum([item.images for item in parsed.items], []): + #resp, content = h.request(image, "GET") + # i = i + 1 #f = open("/home/ppawel/Desktop/test/%s" % i, "w") #f.write(content) #f.close() - else: - pass#print "304: %s" % url return parsed diff --git a/straw/error.py b/straw/error.py index 292cfde..7b5a16f 100644 --- a/straw/error.py +++ b/straw/error.py @@ -34,15 +34,12 @@ def setup_log(): filemode='a') console = logging.StreamHandler() - console.setLevel(logging.INFO) + console.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s') console.setFormatter(formatter) logging.getLogger('').addHandler(console) def debug(msg, *args, **kwargs): - print args - print kwargs - print msg logging.debug(msg) def log(*args): diff --git a/straw/opml.py b/straw/opml.py index d0a9ec8..6c24d5a 100644 --- a/straw/opml.py +++ b/straw/opml.py @@ -45,23 +45,13 @@ class OPMLParseJobHandler(ThreadPoolJobHandler): self.task_class = OPMLParseTaskThread def _split(self): - ti = TaskInfo(0, { "file_path" : self.job.data }) + ti = TaskInfo(0, { "file_path": self.job.data[0], "category": self.job.data[1] }) self.task_queue.put(ti) - - def _prepare_result(self): - result = self.result_queue.get().result - #print "OPMLImport :: imported %d feed(s)" % len(result) - - categories = [] - tree = [] - - #for o in result.roots(): - #print o - # tree.extend(self._build_tree(o)) - - tree = self._build_tree(result.roots()) - return tree + def _prepare_result(self): + task_result = self.result_queue.get() + tree = self._build_tree(task_result.result.roots()) + return (tree, task_result.task_info.data["category"]) def _build_tree(self, outlines, parent = None): save_list = [] @@ -72,7 +62,7 @@ class OPMLParseJobHandler(ThreadPoolJobHandler): # Some feeds exported from Liferea don't have "type" attribute. outline["type"] = "rss" - if outline["type"] == "folder": + if outline["type"] == "folder" or len(outline.children) > 0: category = Category() category.norder = i category.name = outline["text"] @@ -87,15 +77,26 @@ class OPMLParseJobHandler(ThreadPoolJobHandler): else: feed = Feed() feed.norder = i - feed.title = outline["title"] + + if outline.has_key("title"): + feed.title = outline["title"] + elif outline.has_key("text"): + feed.title = outline["text"] + else: + feed.title = "[unknown title]" + feed.parent = parent feed.location = "" if outline.has_key("xmlUrl"): feed.location = outline["xmlUrl"] + elif outline.has_key("url"): + feed.location = outline["url"] if outline.has_key("htmlUrl"): feed.link = outline["htmlUrl"] + elif outline.has_key("url"): + feed.link = outline["url"] else: feed.link = "" @@ -176,6 +177,9 @@ class Outline(dict): def __iter__(self): return self + + def __len__(self): + return len(self._o._children) def next(self): self._index += 1 diff --git a/straw/storage/SQLiteDAO.py b/straw/storage/SQLiteDAO.py index 60f060d..47f8217 100644 --- a/straw/storage/SQLiteDAO.py +++ b/straw/storage/SQLiteDAO.py @@ -35,6 +35,12 @@ class DAO(object): fields.extend(clazz.__bases__[0].persistent_properties) return fields + + def tx_begin(self): + self.storage._tx_begin() + + def tx_commit(self): + self.storage._tx_commit() def save(self, entity): do_insert = entity.id == None @@ -53,7 +59,9 @@ class DAO(object): if do_insert: id = self.storage.insert(entity.__class__.persistent_table, data) - entity.id = id + + if entity.id == None: + entity.id = id else: self.storage.update(entity.__class__.persistent_table, entity.id, data) @@ -106,7 +114,7 @@ class DAO(object): def get_nodes(self, sql = ""): #result = self.storage.query("SELECT f.*, feed_id, unread_count FROM feeds f LEFT JOIN (SELECT COUNT(*) AS unread_count, feed_id FROM items i WHERE i.is_read = 0 GROUP BY i.feed_id) ON feed_id = f.id ORDER BY category_id") - result = self.storage.query("SELECT *, unread_count FROM nodes n LEFT JOIN feeds f USING (id) LEFT JOIN categories c USING (id) LEFT JOIN (SELECT COUNT(*) AS unread_count, feed_id FROM items i WHERE i.is_read = 0 GROUP BY i.feed_id) ON feed_id = f.id ORDER BY n.parent_id, n.norder") + result = self.storage.query("SELECT *, unread_count FROM nodes n LEFT JOIN feeds f ON (f.id = n.id) LEFT JOIN categories c ON (c.id = n.id) LEFT JOIN (SELECT COUNT(*) AS unread_count, feed_id FROM items i WHERE i.is_read = 0 GROUP BY i.feed_id) ON feed_id = f.id ORDER BY n.parent_id, n.norder") entities = [] for row in result: diff --git a/straw/storage/SQLiteStorage.py b/straw/storage/SQLiteStorage.py index 0d86d97..4d7e7a9 100644 --- a/straw/storage/SQLiteStorage.py +++ b/straw/storage/SQLiteStorage.py @@ -1,4 +1,5 @@ from pysqlite2 import dbapi2 as sqlite +import threading DATABASE_FILE_NAME = "data.db" @@ -10,6 +11,7 @@ class SQLiteStorage: def __init__(self, db_path): self._connections = {} + self._txs = {} self._db_path = db_path self._init_db() @@ -39,22 +41,35 @@ class SQLiteStorage: sql = self._get_sql() for statement in sql: c.execute(statement) - self._commit() + self._tx_commit() except Exception, e: print "DB init failed -- %s" % e def _connect(self): - import threading key = threading.currentThread() + if not self._connections.has_key(key): self._connections[key] = sqlite.connect(self._db_path) self._connections[key].row_factory = sqlite.Row return self._connections[key] + + def _tx_begin(self): + key = threading.currentThread() - def _commit(self): + if not self._txs.has_key(key): + self._txs[key] = True + + def _tx_commit(self): self._connect().commit() + key = threading.currentThread() + self._txs[key] = False + + def _in_tx(self): + key = threading.currentThread() + return self._txs.has_key(key) and self._txs[key] + def query(self, query, params = None): if params == None: params = () @@ -82,7 +97,11 @@ class SQLiteStorage: #print query #print data cursor.execute(query, data.values()) - self._commit() + + #print self._in_tx() + if not self._in_tx(): + self._tx_commit() + return cursor.lastrowid def update(self, table, id, data): @@ -103,5 +122,8 @@ class SQLiteStorage: #print query params.append(id) cursor.execute(query, params) - self._commit() + + if not self._in_tx(): + self._tx_commit() + return cursor.lastrowid diff --git a/straw/subscribe.py b/straw/subscribe.py index 61370f6..7a8d3f5 100644 --- a/straw/subscribe.py +++ b/straw/subscribe.py @@ -60,21 +60,16 @@ class SubscribeView(MVP.GladeView): self.category_pixbuf = gtk.Label().render_icon(gtk.STOCK_DIRECTORY, gtk.ICON_SIZE_MENU) - def _populate_tree(self, parent_id, parent = None, path = ""): + def _populate_tree(self, parent_id, parent = None): if not self.nodes.has_key(parent_id): return for node in self.nodes[parent_id]: if node.type == "C": - if path != "": - path = path + " > " + node.title - else: - path = node.title + treeiter = self.category_store.append(parent, [self.category_pixbuf, node.name]) - treeiter = self.category_store.append(parent, [self.category_pixbuf, path]) - - if self.nodes.has_key(node.obj_id): - self._populate_tree(node.obj_id, treeiter, path) + if self.nodes.has_key(node.id): + self._populate_tree(node.id, treeiter) def _on_forward(self): location = self._location_entry.get_text() @@ -127,23 +122,22 @@ class SubscribeView(MVP.GladeView): self._error_box.set_property('visible',False) self._notebook.set_current_page(0) self._window.show() - + + self.store = gtk.ListStore(bool, str, gobject.TYPE_PYOBJECT) + renderer = gtk.CellRendererToggle() + renderer.set_property("activatable", True) column = gtk.TreeViewColumn(_('Keep'), renderer, active = 0) + renderer.connect('toggled', self.col1_toggled_cb, self.store) self._result_tree.append_column(column) #renderer.connect('toggled', self._sticky_toggled) - renderer = gtk.CellRendererText() - column = gtk.TreeViewColumn(_('_Title'), renderer, markup = 1) - self._result_tree.append_column(column) - - self.store = gtk.ListStore(bool, str) + renderer2 = gtk.CellRendererText() + column = gtk.TreeViewColumn(_('_Title'), renderer2, markup = 1) + self._result_tree.append_column(column) self._result_tree.set_model(self.store) - - def setup_category_tree(self): - self.category_store = gtk.TreeStore(gtk.gdk.Pixbuf, str) - + column = gtk.TreeViewColumn() status_renderer = gtk.CellRendererPixbuf() @@ -156,12 +150,21 @@ class SubscribeView(MVP.GladeView): self._category_tree.append_column(column) - self.categories, self.feeds = FeedManager.get_model() - self.nodes = TreeViewManager.get_nodes() + def col1_toggled_cb( self, cell, path, model ): + self.store[path][0] = not self.store[path][0] + + def setup_category_tree(self): + self.category_store = gtk.TreeStore(gtk.gdk.Pixbuf, str) + self.nodes = FeedManager.get_model()[1] self._category_tree.set_model(None) self._category_tree.set_model(self.category_store) + treeiter = self.category_store.append(None, [self.category_pixbuf, "Main category"]) + + self._populate_tree(1, parent = treeiter) + self._category_tree.expand_all() + def display_error(self, text): self._error_box.set_property('visible',True) self._error_text.set_text(text) @@ -189,14 +192,20 @@ class SubscribeView(MVP.GladeView): def _on_button3_clicked(self, *args): if self.state == STATE_FINISH: - pass - + new_feeds = [] + + for feed in [row[2] for row in self.store if row[0]]: + feed.parent = None + new_feeds.append(feed) + + FeedManager.save_all(new_feeds) + def _discovery_finished(self, handler, feeds): gtk.gdk.threads_enter() for feed in feeds: label = xml.sax.saxutils.escape(feed.title) + '\n' + xml.sax.saxutils.escape(feed.location) + '' - self.store.append([False, label]) + self.store.append([None, label, feed]) self._progress_bar.props.visible = False self._button2.set_sensitive(True) @@ -206,8 +215,6 @@ class SubscribeView(MVP.GladeView): self._notebook.set_current_page(1) self.setup_category_tree() - self._populate_tree(1) - self._category_tree.expand_all() gtk.gdk.threads_leave() -- 2.11.4.GIT