From 2f85ecc1fd6c764dfcf34e7c2ecbd116fb0bc761 Mon Sep 17 00:00:00 2001 From: nikosapi Date: Fri, 6 Feb 2009 09:54:28 -0500 Subject: [PATCH] Thomas and Justin's proper file and folder names patch Make the patch work on top of current git HEAD --- src/gpodder/dbsqlite.py | 42 +++++++++++++++++---- src/gpodder/gui.py | 37 ++++++------------ src/gpodder/libgpodder.py | 18 ++++----- src/gpodder/libpodcasts.py | 94 ++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 139 insertions(+), 52 deletions(-) diff --git a/src/gpodder/dbsqlite.py b/src/gpodder/dbsqlite.py index 0f76a8ba..f861f4a6 100644 --- a/src/gpodder/dbsqlite.py +++ b/src/gpodder/dbsqlite.py @@ -164,6 +164,8 @@ class Storage(object): ("etag", "TEXT"), ("deleted", "INTEGER"), ("channel_is_locked", "INTEGER"), + ("foldername", "TEXT"), + ("auto_foldername", "INTEGER") )) self.upgrade_table("episodes", ( @@ -180,14 +182,18 @@ class Storage(object): ("state", "INTEGER"), ("played", "INTEGER"), ("locked", "INTEGER"), + ("filename", "TEXT"), + ("auto_filename", "INTEGER"), )) + cur.execute("""CREATE UNIQUE INDEX IF NOT EXISTS idx_foldername ON channels (foldername)""") cur.execute("""CREATE UNIQUE INDEX IF NOT EXISTS idx_url ON channels (url)""") cur.execute("""CREATE INDEX IF NOT EXISTS idx_sync_to_devices ON channels (sync_to_devices)""") cur.execute("""CREATE INDEX IF NOT EXISTS idx_title ON channels (title)""") cur.execute("""CREATE INDEX IF NOT EXISTS idx_deleted ON channels (deleted)""") cur.execute("""CREATE UNIQUE INDEX IF NOT EXISTS idx_guid ON episodes (guid)""") + cur.execute("""CREATE UNIQUE INDEX IF NOT EXISTS idx_filename ON episodes (filename)""") cur.execute("""CREATE INDEX IF NOT EXISTS idx_channel_id ON episodes (channel_id)""") cur.execute("""CREATE INDEX IF NOT EXISTS idx_pubDate ON episodes (pubDate)""") cur.execute("""CREATE INDEX IF NOT EXISTS idx_state ON episodes (state)""") @@ -256,7 +262,9 @@ class Storage(object): password, last_modified, etag, - channel_is_locked + channel_is_locked, + foldername, + auto_foldername FROM channels WHERE @@ -285,6 +293,8 @@ class Storage(object): 'last_modified': row[12], 'etag': row[13], 'channel_is_locked': row[14], + 'foldername': row[15], + 'auto_foldername': row[16], } if row[0] in stats: @@ -338,10 +348,10 @@ class Storage(object): self.log("save_channel((%s)%s)", c.id or "new", c.url) if c.id is None: - cur.execute("INSERT INTO channels (url, title, override_title, link, description, image, pubDate, sync_to_devices, device_playlist_name, username, password, last_modified, etag, channel_is_locked) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (c.url, c.title, c.override_title, c.link, c.description, c.image, self.__mktime__(c.pubDate), c.sync_to_devices, c.device_playlist_name, c.username, c.password, c.last_modified, c.etag, c.channel_is_locked, )) + cur.execute("INSERT INTO channels (url, title, override_title, link, description, image, pubDate, sync_to_devices, device_playlist_name, username, password, last_modified, etag, channel_is_locked, foldername, auto_foldername) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (c.url, c.title, c.override_title, c.link, c.description, c.image, self.__mktime__(c.pubDate), c.sync_to_devices, c.device_playlist_name, c.username, c.password, c.last_modified, c.etag, c.channel_is_locked, c.foldername, c.auto_foldername, )) self.channel_map[c.url] = cur.lastrowid else: - cur.execute("UPDATE channels SET url = ?, title = ?, override_title = ?, link = ?, description = ?, image = ?, pubDate = ?, sync_to_devices = ?, device_playlist_name = ?, username = ?, password = ?, last_modified = ?, etag = ?, channel_is_locked = ?, deleted = 0 WHERE id = ?", (c.url, c.title, c.override_title, c.link, c.description, c.image, self.__mktime__(c.pubDate), c.sync_to_devices, c.device_playlist_name, c.username, c.password, c.last_modified, c.etag, c.channel_is_locked, c.id, )) + cur.execute("UPDATE channels SET url = ?, title = ?, override_title = ?, link = ?, description = ?, image = ?, pubDate = ?, sync_to_devices = ?, device_playlist_name = ?, username = ?, password = ?, last_modified = ?, etag = ?, channel_is_locked = ?, foldername = ?, auto_foldername = ?, deleted = 0 WHERE id = ?", (c.url, c.title, c.override_title, c.link, c.description, c.image, self.__mktime__(c.pubDate), c.sync_to_devices, c.device_playlist_name, c.username, c.password, c.last_modified, c.etag, c.channel_is_locked, c.foldername, c.auto_foldername, c.id, )) cur.close() self.lock.release() @@ -360,13 +370,13 @@ class Storage(object): del self.channel_map[channel.url] else: cur.execute("UPDATE channels SET deleted = 1 WHERE id = ?", (channel.id, )) - cur.execute("DELETE FROM episodes WHERE channel_id = ? AND state <> ?", (channel.id, self.STATE_DELETED)) + cur.execute("DELETE FROM episodes WHERE channel_id = ? AND state <> ?", (channel.id, self.STATE_DOWNLOADED)) cur.close() self.lock.release() def __read_episodes(self, factory=None, where=None, params=None, commit=True): - sql = "SELECT url, title, length, mimetype, guid, description, link, pubDate, state, played, locked, id FROM episodes" + sql = "SELECT url, title, length, mimetype, guid, description, link, pubDate, state, played, locked, filename, auto_filename, id FROM episodes" if where: sql = "%s %s" % (sql, where) @@ -391,7 +401,9 @@ class Storage(object): 'state': row[8], 'is_played': row[9], 'is_locked': row[10], - 'id': row[11], + 'filename': row[11], + 'auto_filename': row[12], + 'id': row[13], } if episode['state'] is None: episode['state'] = self.STATE_NORMAL @@ -444,10 +456,10 @@ class Storage(object): self.log("save_episode() -- looking up id") if e.id is None: - cur.execute("INSERT INTO episodes (channel_id, url, title, length, mimetype, guid, description, link, pubDate, state, played, locked) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (channel_id, e.url, e.title, e.length, e.mimetype, e.guid, e.description, e.link, self.__mktime__(e.pubDate), e.state, e.is_played, e.is_locked, )) + cur.execute("INSERT INTO episodes (channel_id, url, title, length, mimetype, guid, description, link, pubDate, state, played, locked, filename, auto_filename) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (channel_id, e.url, e.title, e.length, e.mimetype, e.guid, e.description, e.link, self.__mktime__(e.pubDate), e.state, e.is_played, e.is_locked, e.filename, e.auto_filename, )) e.id = cur.lastrowid else: - cur.execute("UPDATE episodes SET title = ?, length = ?, mimetype = ?, description = ?, link = ?, pubDate = ?, state = ?, played = ?, locked = ? WHERE id = ?", (e.title, e.length, e.mimetype, e.description, e.link, self.__mktime__(e.pubDate), e.state, e.is_played, e.is_locked, e.id, )) + cur.execute("UPDATE episodes SET title = ?, length = ?, mimetype = ?, description = ?, link = ?, pubDate = ?, state = ?, played = ?, locked = ?, filename = ?, auto_filename = ? WHERE id = ?", (e.title, e.length, e.mimetype, e.description, e.link, self.__mktime__(e.pubDate), e.state, e.is_played, e.is_locked, e.filename, e.auto_filename, e.id, )) except Exception, e: log('save_episode() failed: %s', e, sender=self) @@ -539,6 +551,20 @@ class Storage(object): log('Could not convert "%s" to a string date.', date) return None + def channel_foldername_exists(self, foldername): + """ + Returns True if a foldername for a channel exists. + False otherwise. + """ + return self.__get__("SELECT id FROM channels WHERE foldername = ?", (foldername,)) is not None + + def episode_filename_exists(self, filename): + """ + Returns True if a filename for an episode exists. + False otherwise. + """ + return self.__get__("SELECT id FROM episodes WHERE filename = ?", (filename,)) is not None + def find_channel_id(self, url): """ Looks up the channel id in the map (which lists all undeleted diff --git a/src/gpodder/gui.py b/src/gpodder/gui.py index fc8f8a75..cb8b7560 100644 --- a/src/gpodder/gui.py +++ b/src/gpodder/gui.py @@ -669,9 +669,6 @@ class gPodder(GladeWidget): self.user_apps_reader = UserAppsReader(['audio', 'video']) Thread(target=self.read_apps).start() - # Clean up old, orphaned download files - gl.clean_up_downloads( delete_partial = True) - # Set the "Device" menu item for the first time self.update_item_device() @@ -692,6 +689,9 @@ class gPodder(GladeWidget): self.feed_cache_update_cancelled = False self.update_feed_cache(force_update=gl.config.update_on_startup) + # Clean up old, orphaned download files + gl.clean_up_downloads(delete_partial=True) + # Start the auto-update procedure self.auto_update_procedure(first_run=True) @@ -2483,33 +2483,18 @@ class gPodder(GladeWidget): gPodderChannel(channel=self.active_channel, callback_closed=lambda: self.updateComboBox(only_selected_channel=True), callback_change_url=self.change_channel_url) def change_channel_url(self, old_url, new_url): - channel = None - try: - channel = podcastChannel.load(url=new_url, create=True) - except: - channel = None - - if channel is None: - self.show_message(_('The specified URL is invalid. The old URL has been used instead.'), _('Invalid URL')) - return - for channel in self.channels: if channel.url == old_url: log('=> change channel url from %s to %s', old_url, new_url) - old_save_dir = channel.save_dir channel.url = new_url - new_save_dir = channel.save_dir - log('old save dir=%s', old_save_dir, sender=self) - log('new save dir=%s', new_save_dir, sender=self) - files = glob.glob(os.path.join(old_save_dir, '*')) - log('moving %d files to %s', len(files), new_save_dir, sender=self) - for file in files: - log('moving %s', file, sender=self) - shutil.move(file, new_save_dir) - try: - os.rmdir(old_save_dir) - except: - log('Warning: cannot delete %s', old_save_dir, sender=self) + # remove etag and last_modified to force an update + channel.etag = '' + channel.last_modified = '' + (success, error) = channel.update() + if not success: + self.show_message(_('The specified URL is invalid. The old URL has been used instead.'), _('Invalid URL')) + channel.url = old_url + break save_channels(self.channels) # update feed cache and select the podcast with the new URL afterwards diff --git a/src/gpodder/libgpodder.py b/src/gpodder/libgpodder.py index 03cff6b6..8783dc47 100644 --- a/src/gpodder/libgpodder.py +++ b/src/gpodder/libgpodder.py @@ -297,18 +297,14 @@ class gPodderLib(object): for tempfile in temporary_files: util.delete_file(tempfile) - # Clean up empty download folders - download_dirs = glob.glob( '%s/*' % ( self.downloaddir, )) + # Clean up empty download folders and abandoned download folders + download_dirs = glob.glob(os.path.join(self.downloaddir, '*')) for ddir in download_dirs: - if os.path.isdir( ddir): - globr = glob.glob( '%s/*' % ( ddir, )) - if not globr: - log( 'Stale download directory found: %s', os.path.basename( ddir)) - try: - os.rmdir( ddir) - log( 'Successfully removed %s.', ddir) - except: - log( 'Could not remove %s.', ddir) + if os.path.isdir(ddir) and not db.channel_foldername_exists(os.path.basename(ddir)): + globr = glob.glob(os.path.join(ddir, '*')) + if len(globr) == 0 or (len(globr) == 1 and globr[0].endswith('/cover')): + log('Stale download directory found: %s', os.path.basename(ddir), sender=self) + shutil.rmtree(ddir, ignore_errors=True) def get_download_dir( self): util.make_directory( self.config.download_dir) diff --git a/src/gpodder/libpodcasts.py b/src/gpodder/libpodcasts.py index 3b0ebae1..7c855d64 100644 --- a/src/gpodder/libpodcasts.py +++ b/src/gpodder/libpodcasts.py @@ -81,6 +81,7 @@ class HTTPAuthError(Exception): pass class podcastChannel(object): """holds data for a complete channel""" SETTINGS = ('sync_to_devices', 'device_playlist_name','override_title','username','password') + MAX_FOLDERNAME_LENGTH = 150 icon_cache = {} fc = cache.Cache() @@ -236,6 +237,8 @@ class podcastChannel(object): self.newest_pubdate_cached = None self.update_flag = False # channel is updating or to be updated self.iter = None + self.foldername = None + self.auto_foldername = 1 # automatically generated foldername # should this channel be synced to devices? (ex: iPod) self.sync_to_devices = True @@ -265,12 +268,6 @@ class podcastChannel(object): def update_save_dir_size(self): self.save_dir_size = util.calculate_size(self.save_dir) - - def get_filename( self): - """Return the MD5 sum of the channel URL""" - return hashlib.md5( self.url).hexdigest() - - filename = property(fget=get_filename) def get_title( self): if self.override_title: @@ -289,6 +286,30 @@ class podcastChannel(object): def set_custom_title( self, custom_title): custom_title = custom_title.strip() + # make sure self.foldername is initialized + self.get_save_dir() + + # rename folder if custom_title looks sane + new_folder_name = self.find_unique_folder_name(custom_title) + if len(new_folder_name) > 0 and new_folder_name != self.foldername: + log('Changing foldername based on custom title: %s', custom_title, sender=self) + new_folder = os.path.join(gl.downloaddir, new_folder_name) + old_folder = os.path.join(gl.downloaddir, self.foldername) + if os.path.exists(old_folder): + if not os.path.exists(new_folder): + # Old folder exists, new folder does not -> simply rename + log('Renaming %s => %s', old_folder, new_folder, sender=self) + os.rename(old_folder, new_folder) + else: + # Both folders exist -> move files and delete old folder + log('Moving files from %s to %s', old_folder, new_folder, sender=self) + for file in glob.glob(os.path.join(old_folder, '*')): + shutil.move(file, new_folder) + log('Removing %s', old_folder, sender=self) + shutil.rmtree(old_folder, ignore_errors=True) + self.foldername = new_folder_name + self.save() + if custom_title != self.__title: self.override_title = custom_title else: @@ -415,8 +436,65 @@ class podcastChannel(object): def find_episode( self, url): return db.load_episode(url, factory=lambda x: podcastItem.create_from_dict(x, self)) + @classmethod + def find_unique_folder_name(cls, foldername): + current_try = util.sanitize_filename(foldername, cls.MAX_FOLDERNAME_LENGTH) + next_try_id = 2 + + while db.channel_foldername_exists(current_try) and \ + not os.path.exists(os.path.join(gl.downloaddir, current_try)): + current_try = '%s (%d)' % (foldername, next_try_id) + next_try_id += 1 + + return current_try + def get_save_dir(self): - save_dir = os.path.join(gl.downloaddir, self.filename, '') + urldigest = hashlib.md5(self.url).hexdigest() + sanitizedurl = util.sanitize_filename(self.url, self.MAX_FOLDERNAME_LENGTH) + if self.foldername is None or (self.auto_foldername and (self.foldername == urldigest or self.foldername == sanitizedurl)): + # we must change the folder name, because it has not been set manually + fn_template = util.sanitize_filename(self.title, self.MAX_FOLDERNAME_LENGTH) + + # if this is an empty string, try the basename + if len(fn_template) == 0: + log('That is one ugly feed you have here! (Report this to bugs.gpodder.org: %s)', self.url, sender=self) + fn_template = util.sanitize_filename(os.path.basename(self.url), self.MAX_FOLDERNAME_LENGTH) + + # If the basename is also empty, use the first 6 md5 hexdigest chars of the URL + if len(fn_template) == 0: + log('That is one REALLY ugly feed you have here! (Report this to bugs.gpodder.org: %s)', self.url, sender=self) + fn_template = urldigest # no need for sanitize_filename here + + # Find a unique folder name for this podcast + wanted_foldername = self.find_unique_folder_name(fn_template) + + # if the foldername has not been set, check if the (old) md5 filename exists + if self.foldername is None and os.path.exists(os.path.join(gl.downloaddir, urldigest)): + log('Found pre-0.14.0 download folder for %s: %s', self.title, urldigest, sender=self) + self.foldername = urldigest + + # we have a valid, new folder name in "current_try" -> use that! + if self.foldername is not None and wanted_foldername != self.foldername: + # there might be an old download folder crawling around - move it! + new_folder_name = os.path.join(gl.downloaddir, wanted_foldername) + old_folder_name = os.path.join(gl.downloaddir, self.foldername) + if os.path.exists(old_folder_name): + if not os.path.exists(new_folder_name): + # Old folder exists, new folder does not -> simply rename + log('Renaming %s => %s', old_folder_name, new_folder_name, sender=self) + os.rename(old_folder_name, new_folder_name) + else: + # Both folders exist -> move files and delete old folder + log('Moving files from %s to %s', old_folder_name, new_folder_name, sender=self) + for file in glob.glob(os.path.join(old_folder_name, '*')): + shutil.move(file, new_folder_name) + log('Removing %s', old_folder_name, sender=self) + shutil.rmtree(old_folder_name, ignore_errors=True) + log('Updating foldername of %s to "%s".', self.url, wanted_foldername, sender=self) + self.foldername = wanted_foldername + self.save() + + save_dir = os.path.join(gl.downloaddir, self.foldername) # Create save_dir if it does not yet exist if not util.make_directory( save_dir): @@ -545,6 +623,8 @@ class podcastItem(object): self.link = '' self.channel = channel self.pubDate = 0 + self.filename = None + self.auto_filename = 1 # automatically generated filename self.state = db.STATE_NORMAL self.is_played = False -- 2.11.4.GIT