src/gpodder/plugins/soundcloud.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # gPodder - A media aggregator and podcast client
   4 # Copyright (c) 2005-2018 The gPodder Team
   5 #
   6 # gPodder is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # gPodder is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 # Soundcloud.com API client module for gPodder
  21 # Thomas Perl <thp@gpodder.org>; 2009-11-03
  22
  23 import json
  24 import logging
  25 import os
  26 import re
  27 import time
  28 import urllib.error
  29 import urllib.parse
  30 import urllib.request
  31
  32 import gpodder
  33 from gpodder import feedcore, model, registry, util
  34
  35 _ = gpodder.gettext
  36
  37
  38 # gPodder's consumer key for the Soundcloud API
  39 CONSUMER_KEY = 'zrweghtEtnZLpXf3mlm8mQ'
  40
  41
  42 logger = logging.getLogger(__name__)
  43
  44
  45 def soundcloud_parsedate(s):
  46     """Parse a string into a unix timestamp
  47
  48     Only strings provided by Soundcloud's API are
  49     parsed with this function (2009/11/03 13:37:00).
  50     """
  51     m = re.match(r'(\d{4})/(\d{2})/(\d{2}) (\d{2}):(\d{2}):(\d{2})', s)
  52     return time.mktime(tuple([int(x) for x in m.groups()] + [0, 0, -1]))
  53
  54
  55 def get_metadata(url):
  56     """Get file download metadata
  57
  58     Returns a (size, type, name) from the given download
  59     URL. Will use the network connection to determine the
  60     metadata via the HTTP header fields.
  61     """
  62     track_response = util.urlopen(url)
  63     filesize = track_response.headers['content-length'] or '0'
  64     filetype = track_response.headers['content-type'] or 'application/octet-stream'
  65     filename = util.get_header_param(track_response.headers, 'filename', 'content-disposition') \
  66         or os.path.basename(os.path.dirname(url))
  67     track_response.close()
  68     return filesize, filetype, filename
  69
  70
  71 class SoundcloudUser(object):
  72     def __init__(self, username):
  73         self.username = username
  74         self.cache_file = os.path.join(gpodder.home, 'Soundcloud')
  75         if os.path.exists(self.cache_file):
  76             try:
  77                 self.cache = json.load(open(self.cache_file, 'r'))
  78             except:
  79                 self.cache = {}
  80         else:
  81             self.cache = {}
  82
  83     def commit_cache(self):
  84         json.dump(self.cache, open(self.cache_file, 'w'))
  85
  86     def get_user_info(self):
  87         global CONSUMER_KEY
  88         key = ':'.join((self.username, 'user_info'))
  89         if key in self.cache:
  90             if self.cache[key].get('code', 200) == 200:
  91                 return self.cache[key]
  92
  93         try:
  94             # find user ID in soundcloud page
  95             url = 'https://soundcloud.com/' + self.username
  96             r = util.urlopen(url)
  97             if not r.ok:
  98                 raise Exception('Soundcloud "%s": %d %s' % (url, r.status_code, r.reason))
  99             uid = re.search(r'"https://api.soundcloud.com/users/([0-9]+)"', r.text)
 100             if not uid:
 101                 raise Exception('Soundcloud user ID not found for "%s"' % url)
 102             uid = int(uid.group(1))
 103
 104             # load user info API
 105             json_url = 'https://api.soundcloud.com/users/%d.json?consumer_key=%s' % (uid, CONSUMER_KEY)
 106             r = util.urlopen(json_url)
 107             if not r.ok:
 108                 raise Exception('Soundcloud "%s": %d %s' % (json_url, r.status_code, r.reason))
 109             user_info = json.loads(r.text)
 110             if user_info.get('code', 200) != 200:
 111                 raise Exception('Soundcloud "%s": %s' % (json_url, user_info.get('message', '')))
 112
 113             self.cache[key] = user_info
 114         finally:
 115             self.commit_cache()
 116
 117         return user_info
 118
 119     def get_coverart(self):
 120         user_info = self.get_user_info()
 121         return user_info.get('avatar_url', None)
 122
 123     def get_user_id(self):
 124         user_info = self.get_user_info()
 125         return user_info.get('id', None)
 126
 127     def get_tracks(self, feed):
 128         """Get a generator of tracks from a SC user
 129
 130         The generator will give you a dictionary for every
 131         track it can find for its user."""
 132         global CONSUMER_KEY
 133         try:
 134             json_url = ('https://api.soundcloud.com/users/%(user)s/%(feed)s.'
 135                         'json?consumer_key=%'
 136                         '(consumer_key)s&limit=200'
 137                         % {"user": self.get_user_id(),
 138                            "feed": feed,
 139                            "consumer_key": CONSUMER_KEY})
 140             logger.debug("loading %s", json_url)
 141
 142             json_tracks = util.urlopen(json_url).json()
 143             tracks = [track for track in json_tracks if track['streamable'] or track['downloadable']]
 144             total_count = len(json_tracks)
 145
 146             if len(tracks) == 0 and total_count > 0:
 147                 logger.warning("Download of all %i %s of user %s is disabled" %
 148                             (total_count, feed, self.username))
 149             else:
 150                 logger.info("%i/%i downloadable tracks for user %s %s feed" %
 151                             (len(tracks), total_count, self.username, feed))
 152
 153             for track in tracks:
 154                 # Prefer stream URL (MP3), fallback to download URL
 155                 base_url = track.get('stream_url') if track['streamable'] else track['download_url']
 156                 url = base_url + '?consumer_key=' + CONSUMER_KEY
 157                 if url not in self.cache:
 158                     try:
 159                         self.cache[url] = get_metadata(url)
 160                     except:
 161                         continue
 162                 filesize, filetype, filename = self.cache[url]
 163
 164                 yield {
 165                     'title': track.get('title', track.get('permalink')) or _('Unknown track'),
 166                     'link': track.get('permalink_url') or 'https://soundcloud.com/' + self.username,
 167                     'description': util.remove_html_tags(track.get('description') or ''),
 168                     'description_html': '',
 169                     'url': url,
 170                     'file_size': int(filesize),
 171                     'mime_type': filetype,
 172                     'guid': str(track.get('permalink', track.get('id'))),
 173                     'published': soundcloud_parsedate(track.get('created_at', None)),
 174                 }
 175         finally:
 176             self.commit_cache()
 177
 178
 179 class SoundcloudFeed(model.Feed):
 180     URL_REGEX = re.compile(r'https?://([a-z]+\.)?soundcloud\.com/([^/]+)$', re.I)
 181
 182     @classmethod
 183     def fetch_channel(cls, channel, max_episodes=0):
 184         url = channel.authenticate_url(channel.url)
 185         return cls.handle_url(url, max_episodes)
 186
 187     @classmethod
 188     def handle_url(cls, url, max_episodes):
 189         m = cls.URL_REGEX.match(url)
 190         if m is not None:
 191             subdomain, username = m.groups()
 192             return feedcore.Result(feedcore.UPDATED_FEED, cls(username, max_episodes))
 193
 194     def __init__(self, username, max_episodes):
 195         self.username = username
 196         self.sc_user = SoundcloudUser(username)
 197         self.max_episodes = max_episodes
 198
 199     def get_title(self):
 200         return _('%s on Soundcloud') % self.username
 201
 202     def get_cover_url(self):
 203         return self.sc_user.get_coverart()
 204
 205     def get_link(self):
 206         return 'https://soundcloud.com/%s' % self.username
 207
 208     def get_description(self):
 209         return _('Tracks published by %s on Soundcloud.') % self.username
 210
 211     def get_new_episodes(self, channel, existing_guids):
 212         return self._get_new_episodes(channel, existing_guids, 'tracks')
 213
 214     def get_next_page(self, channel, max_episodes=0):
 215         # one could return more, but it would consume too many api calls
 216         # (see PR #184)
 217         return None
 218
 219     def _get_new_episodes(self, channel, existing_guids, track_type):
 220         tracks = list(self.sc_user.get_tracks(track_type))
 221         if self.max_episodes > 0:
 222             tracks = tracks[:self.max_episodes]
 223
 224         seen_guids = {track['guid'] for track in tracks}
 225         episodes = []
 226
 227         for track in tracks:
 228             if track['guid'] not in existing_guids:
 229                 episode = channel.episode_factory(track)
 230                 episode.save()
 231                 episodes.append(episode)
 232
 233         return episodes, seen_guids
 234
 235
 236 class SoundcloudFavFeed(SoundcloudFeed):
 237     URL_REGEX = re.compile(r'https?://([a-z]+\.)?soundcloud\.com/([^/]+)/favorites', re.I)
 238
 239     def __init__(self, username):
 240         super(SoundcloudFavFeed, self).__init__(username)
 241
 242     def get_title(self):
 243         return _("%s's favorites on Soundcloud") % self.username
 244
 245     def get_link(self):
 246         return 'https://soundcloud.com/%s/favorites' % self.username
 247
 248     def get_description(self):
 249         return _('Tracks favorited by %s on Soundcloud.') % self.username
 250
 251     def get_new_episodes(self, channel, existing_guids):
 252         return self._get_new_episodes(channel, existing_guids, 'favorites')
 253
 254
 255 # Register our URL handlers
 256 registry.feed_handler.register(SoundcloudFeed.fetch_channel)
 257 registry.feed_handler.register(SoundcloudFavFeed.fetch_channel)
 258
 259
 260 def search_for_user(query):
 261     json_url = 'https://api.soundcloud.com/users.json?q=%s&consumer_key=%s' % (urllib.parse.quote(query), CONSUMER_KEY)
 262     return util.urlopen(json_url).json()