From 4113c4737c68174065cce2cde114271febd83e41 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Stefan=20K=C3=B6gl?= Date: Sun, 31 May 2015 17:13:38 +0200 Subject: [PATCH] Fix Python 2 / 3 incompatabilities --- mygpo/api/__init__.py | 2 +- mygpo/api/advanced/lists.py | 4 +- mygpo/api/basic_auth.py | 5 +- mygpo/api/opml.py | 4 +- mygpo/api/simple.py | 3 +- mygpo/flattr.py | 2 +- mygpo/podcastlists/tests.py | 2 +- mygpo/podcasts/models.py | 5 +- mygpo/search/index.py | 4 +- mygpo/test.py | 7 +- mygpo/users/models.py | 6 +- mygpo/usersettings/tests.py | 2 +- mygpo/utils.py | 166 +++----------------------------------------- mygpo/web/utils.py | 41 +++++++---- mygpo/web/views/podcast.py | 3 +- requirements.txt | 2 +- 16 files changed, 61 insertions(+), 197 deletions(-) diff --git a/mygpo/api/__init__.py b/mygpo/api/__init__.py index a5e1ccc7..1b9bc1d0 100644 --- a/mygpo/api/__init__.py +++ b/mygpo/api/__init__.py @@ -76,7 +76,7 @@ class APIView(View): def get_since(self, request): """ Returns parsed "since" GET parameter """ - since_ = request.GET.get('since', None) + since_ = int(request.GET.get('since', None)) if since_ is None: raise RequestException("parameter 'since' missing") diff --git a/mygpo/api/advanced/lists.py b/mygpo/api/advanced/lists.py index 8bc244d9..2e670d26 100644 --- a/mygpo/api/advanced/lists.py +++ b/mygpo/api/advanced/lists.py @@ -77,7 +77,7 @@ def create(request, username, format): if not created: return HttpResponse('List already exists', status=409) - urls = parse_subscription(request.body, format) + urls = parse_subscription(request.body.decode('utf-8'), format) podcasts = [Podcast.objects.get_or_create_for_url(url) for url in urls] for podcast in podcasts: @@ -164,7 +164,7 @@ def get_list(request, plist, owner, format): @cors_origin() def update_list(request, plist, owner, format): """ Replaces the podcasts in the list and returns 204 No Content """ - urls = parse_subscription(request.body, format) + urls = parse_subscription(request.body.decode('utf-8'), format) podcasts = [Podcast.objects.get_or_create_for_url(url) for url in urls] plist.set_entries(podcasts) diff --git a/mygpo/api/basic_auth.py b/mygpo/api/basic_auth.py index 1a4896f7..d420bd46 100644 --- a/mygpo/api/basic_auth.py +++ b/mygpo/api/basic_auth.py @@ -15,6 +15,7 @@ # along with my.gpodder.org. If not, see . # +import base64 from functools import wraps from django.http import HttpResponse, HttpResponseBadRequest @@ -58,7 +59,9 @@ def view_or_basicauth(view, request, test_func, realm = "", *args, **kwargs): # NOTE: We are only support basic authentication for now. if auth_type.lower() == 'basic': try: - credentials = credentials.decode('base64').split(':', 1) + credentials = base64.b64decode(credentials)\ + .decode('utf-8')\ + .split(':', 1) except UnicodeDecodeError as e: return HttpResponseBadRequest( diff --git a/mygpo/api/opml.py b/mygpo/api/opml.py index 2ee6d422..02a46329 100644 --- a/mygpo/api/opml.py +++ b/mygpo/api/opml.py @@ -25,7 +25,7 @@ the web and to export a list of podcast objects to valid OPML 1.1 files. import os import xml.dom.minidom -import email.Utils +import email.utils class Importer(object): @@ -72,7 +72,7 @@ class Exporter(object): def __init__(self, title='my.gpodder.org Subscriptions'): self.title = title - self.created = email.Utils.formatdate(localtime=True) + self.created = email.utils.formatdate(localtime=True) def generate(self, channels): """ diff --git a/mygpo/api/simple.py b/mygpo/api/simple.py index d2487349..94fe443f 100644 --- a/mygpo/api/simple.py +++ b/mygpo/api/simple.py @@ -75,7 +75,8 @@ def subscriptions(request, username, device_uid, format): elif request.method in ('PUT', 'POST'): try: - subscriptions = parse_subscription(request.body, format) + body = request.body.decode('utf-8') + subscriptions = parse_subscription(body, format) except JSONDecodeError as e: return HttpResponseBadRequest('Unable to parse POST data: %s' % str(e)) diff --git a/mygpo/flattr.py b/mygpo/flattr.py index 12b2f3e5..88b0f0ff 100644 --- a/mygpo/flattr.py +++ b/mygpo/flattr.py @@ -117,7 +117,7 @@ class Flattr(object): if not self.user.profile.settings.get_wksetting(FLATTR_TOKEN): return (0, False) - quote_url = urllib.parse.quote_plus(utils.sanitize_encoding(payment_url)) + quote_url = urllib.parse.quote_plus(payment_url) url = self.THING_INFO_URL_TEMPLATE % {'url': quote_url} data = self.request(url) return (int(data.get('flattrs', 0)), bool(data.get('flattred', False))) diff --git a/mygpo/podcastlists/tests.py b/mygpo/podcastlists/tests.py index eb7defdb..75ca24f6 100644 --- a/mygpo/podcastlists/tests.py +++ b/mygpo/podcastlists/tests.py @@ -88,7 +88,7 @@ class TestAPI(TestCase): # assert that the list has actually been updated resp = self.client.get(url, content_type="text/plain", **self.extra) - resp_urls = [_f for _f in resp.content.split('\n') if _f] + resp_urls = [u for u in resp.content.decode('utf-8').split('\n') if u] self.assertEqual(urls2, resp_urls) # delete the list diff --git a/mygpo/podcasts/models.py b/mygpo/podcasts/models.py index 37e2968f..dab27363 100644 --- a/mygpo/podcasts/models.py +++ b/mygpo/podcasts/models.py @@ -37,9 +37,6 @@ class TitleModel(models.Model): subtitle = models.TextField(null=False, blank=True) def __str__(self): - return self.title.encode('ascii', errors='replace') - - def __unicode(self): return self.title class Meta: @@ -543,7 +540,7 @@ class Podcast(UUIDModel, TitleModel, DescriptionModel, LinkModel, if not self.url: logger.warn('Podcast with ID {podcast_id} does not have a URL' - .format(podcast_id=self.id.hex)) + .format(podcast_id=self.id)) return _('Unknown Podcast') return _('Unknown Podcast from {domain}'.format( diff --git a/mygpo/search/index.py b/mygpo/search/index.py index cbcd856e..6e7b8193 100644 --- a/mygpo/search/index.py +++ b/mygpo/search/index.py @@ -25,7 +25,7 @@ def index_podcast(sender, **kwargs): conn = get_connection() podcast = kwargs['instance'] - logger.info('Indexing podcast %s', podcast) + logger.info('Indexing podcast {0}', podcast) document = podcast_to_json(podcast) @@ -40,7 +40,7 @@ def create_index(): """ Creates the Elasticsearch index """ conn = get_connection() - logger.info('Creating index %s' % settings.ELASTICSEARCH_INDEX) + logger.info('Creating index {0}', settings.ELASTICSEARCH_INDEX) try: conn.indices.create_index(settings.ELASTICSEARCH_INDEX) diff --git a/mygpo/test.py b/mygpo/test.py index 692260a2..322296de 100644 --- a/mygpo/test.py +++ b/mygpo/test.py @@ -1,4 +1,5 @@ import os.path +import base64 from django.core.urlresolvers import resolve from django.contrib.auth.models import AnonymousUser @@ -9,9 +10,9 @@ from mygpo.utils import random_token def create_auth_string(username, password): - import base64 - credentials = base64.encodestring("%s:%s" % (username, password)).rstrip() - auth_string = 'Basic %s' % credentials + pwdstr = '{0}:{1}'.format(username, password).rstrip() + credentials = base64.b64encode(pwdstr.encode('utf-8')) + auth_string = 'Basic ' + credentials.decode('ascii') return auth_string diff --git a/mygpo/users/models.py b/mygpo/users/models.py index da6ce3ac..702e93f3 100644 --- a/mygpo/users/models.py +++ b/mygpo/users/models.py @@ -343,11 +343,7 @@ class Client(UUIDModel, DeleteableModel): return self.name or self.uid def __str__(self): - return '{} ({})'.format(self.name.encode('ascii', errors='replace'), - self.uid.encode('ascii', errors='replace')) - - def __unicode__(self): - return '{} ({})'.format(self.name, self.uid) + return '{name} ({uid})'.format(name=self.name, uid=self.uid) TOKEN_NAMES = ('subscriptions_token', 'favorite_feeds_token', diff --git a/mygpo/usersettings/tests.py b/mygpo/usersettings/tests.py index 8a78d18d..4edaca4a 100644 --- a/mygpo/usersettings/tests.py +++ b/mygpo/usersettings/tests.py @@ -84,7 +84,7 @@ class TestAPI(TestCase): # get settings resp = self.client.get(url, **self.extra) self.assertEqual(resp.status_code, 200, resp.content) - self.assertEqual(json.loads(resp.content), {'a': 'x'}) + self.assertEqual(json.loads(resp.content.decode('utf-8')), {'a': 'x'}) def get_url(self, username, scope, params={}): url = reverse('settings-api', kwargs={ diff --git a/mygpo/utils.py b/mygpo/utils.py index 8d807070..62a8208f 100644 --- a/mygpo/utils.py +++ b/mygpo/utils.py @@ -46,8 +46,8 @@ logger = logging.getLogger(__name__) def daterange(from_date, to_date=None, leap=timedelta(days=1)): """ - >>> from_d = datetime(2010, 01, 01) - >>> to_d = datetime(2010, 01, 05) + >>> from_d = datetime(2010, 1, 1) + >>> to_d = datetime(2010, 1, 5) >>> list(daterange(from_d, to_d)) [datetime.datetime(2010, 1, 1, 0, 0), datetime.datetime(2010, 1, 2, 0, 0), datetime.datetime(2010, 1, 3, 0, 0), datetime.datetime(2010, 1, 4, 0, 0), datetime.datetime(2010, 1, 5, 0, 0)] """ @@ -287,18 +287,20 @@ def parse_range(s, min, max, default=None): >>> parse_range('5', 0, 10) 5 - >>> parse_range('0', 5, 10) - 5 + >>> parse_range('0', 5.0, 10) + 5.0 >>> parse_range('15',0, 10) 10 - >>> parse_range('x', 0, 20) - 10 + >>> parse_range('x', 0., 20) + 10.0 >>> parse_range('x', 0, 20, 20) 20 """ + out_type = type(min) + try: val = int(s) if val < min: @@ -308,7 +310,7 @@ def parse_range(s, min, max, default=None): return val except (ValueError, TypeError): - return default if default is not None else (max-min)/2 + return default if default is not None else out_type((max-min)/2) @@ -586,7 +588,7 @@ def username_password_from_url(url): >>> username_password_from_url('http://i%2Fo:P%40ss%3A@host.com/') ('i/o', 'P@ss:') >>> username_password_from_url('ftp://%C3%B6sterreich@host.com/') - ('\xc3\xb6sterreich', None) + ('österreich', None) >>> username_password_from_url('http://w%20x:y%20z@example.org/') ('w x', 'y z') >>> username_password_from_url('http://example.com/x@y:z@test.com/') @@ -662,26 +664,6 @@ def url_strip_authentication(url): # Native filesystem encoding detection encoding = sys.getfilesystemencoding() -def sanitize_encoding(filename): - r""" - Generate a sanitized version of a string (i.e. - remove invalid characters and encode in the - detected native language encoding). - - >>> sanitize_encoding('\x80') - '' - >>> sanitize_encoding(u'unicode') - 'unicode' - """ - # The encoding problem goes away in Python 3.. hopefully! - if sys.version_info >= (3, 0): - return filename - - global encoding - if not isinstance(filename, str): - filename = filename.decode(encoding, 'ignore') - return filename.encode(encoding, 'ignore') - def get_git_head(): """ returns the commit and message of the current git HEAD """ @@ -706,131 +688,6 @@ def get_git_head(): return commit, msg - -# https://gist.github.com/samuraisam/901117 - -default_fudge = timedelta(seconds=0, microseconds=0, days=0) - -def deep_eq(_v1, _v2, datetime_fudge=default_fudge, _assert=False): - """ - Tests for deep equality between two python data structures recursing - into sub-structures if necessary. Works with all python types including - iterators and generators. This function was dreampt up to test API responses - but could be used for anything. Be careful. With deeply nested structures - you may blow the stack. - - Options: - datetime_fudge => this is a datetime.timedelta object which, when - comparing dates, will accept values that differ - by the number of seconds specified - _assert => passing yes for this will raise an assertion error - when values do not match, instead of returning - false (very useful in combination with pdb) - - Doctests included: - - >>> x1, y1 = ({'a': 'b'}, {'a': 'b'}) - >>> deep_eq(x1, y1) - True - >>> x2, y2 = ({'a': 'b'}, {'b': 'a'}) - >>> deep_eq(x2, y2) - False - >>> x3, y3 = ({'a': {'b': 'c'}}, {'a': {'b': 'c'}}) - >>> deep_eq(x3, y3) - True - >>> x4, y4 = ({'c': 't', 'a': {'b': 'c'}}, {'a': {'b': 'n'}, 'c': 't'}) - >>> deep_eq(x4, y4) - False - >>> x5, y5 = ({'a': [1,2,3]}, {'a': [1,2,3]}) - >>> deep_eq(x5, y5) - True - >>> x6, y6 = ({'a': [1,'b',8]}, {'a': [2,'b',8]}) - >>> deep_eq(x6, y6) - False - >>> x7, y7 = ('a', 'a') - >>> deep_eq(x7, y7) - True - >>> x8, y8 = (['p','n',['asdf']], ['p','n',['asdf']]) - >>> deep_eq(x8, y8) - True - >>> x9, y9 = (['p','n',['asdf',['omg']]], ['p', 'n', ['asdf',['nowai']]]) - >>> deep_eq(x9, y9) - False - >>> x10, y10 = (1, 2) - >>> deep_eq(x10, y10) - False - >>> deep_eq((str(p) for p in xrange(10)), (str(p) for p in xrange(10))) - True - >>> str(deep_eq(range(4), range(4))) - 'True' - >>> deep_eq(xrange(100), xrange(100)) - True - >>> deep_eq(xrange(2), xrange(5)) - False - >>> from datetime import datetime, timedelta - >>> d1, d2 = (datetime.utcnow(), datetime.utcnow() + timedelta(seconds=4)) - >>> deep_eq(d1, d2) - False - >>> deep_eq(d1, d2, datetime_fudge=timedelta(seconds=5)) - True - """ - _deep_eq = functools.partial(deep_eq, datetime_fudge=datetime_fudge, - _assert=_assert) - - def _check_assert(R, a, b, reason=''): - if _assert and not R: - assert 0, "an assertion has failed in deep_eq (%s) %s != %s" % ( - reason, str(a), str(b)) - return R - - def _deep_dict_eq(d1, d2): - k1, k2 = (sorted(d1.keys()), sorted(d2.keys())) - if k1 != k2: # keys should be exactly equal - return _check_assert(False, k1, k2, "keys") - - return _check_assert(operator.eq(sum(_deep_eq(d1[k], d2[k]) - for k in k1), - len(k1)), d1, d2, "dictionaries") - - def _deep_iter_eq(l1, l2): - if len(l1) != len(l2): - return _check_assert(False, l1, l2, "lengths") - return _check_assert(operator.eq(sum(_deep_eq(v1, v2) - for v1, v2 in zip(l1, l2)), - len(l1)), l1, l2, "iterables") - - def op(a, b): - _op = operator.eq - if type(a) == datetime and type(b) == datetime: - s = datetime_fudge.seconds - t1, t2 = (time.mktime(a.timetuple()), time.mktime(b.timetuple())) - l = t1 - t2 - l = -l if l > 0 else l - return _check_assert((-s if s > 0 else s) <= l, a, b, "dates") - return _check_assert(_op(a, b), a, b, "values") - - c1, c2 = (_v1, _v2) - - # guard against strings because they are iterable and their - # elements yield iterables infinitely. - # I N C E P T I O N - for t in str: - if isinstance(_v1, t): - break - else: - if isinstance(_v1, dict): - op = _deep_dict_eq - else: - try: - c1, c2 = (list(iter(_v1)), list(iter(_v2))) - except TypeError: - c1, c2 = _v1, _v2 - else: - op = _deep_iter_eq - - return op(c1, c2) - - def parse_request_body(request): """ returns the parsed request body, handles gzip encoding """ @@ -906,9 +763,6 @@ def normalize_feed_url(url): if not url or len(url) < 8: return None - if isinstance(url, str): - url = url.encode('utf-8', 'ignore') - # This is a list of prefixes that you can use to minimize the amount of # keystrokes that you have to use. # Feel free to suggest other useful prefixes, and I'll add them here. diff --git a/mygpo/web/utils.py b/mygpo/web/utils.py index af9a1343..4549ca6c 100644 --- a/mygpo/web/utils.py +++ b/mygpo/web/utils.py @@ -1,4 +1,7 @@ +from __future__ import division + import re +import math import string import collections from datetime import datetime @@ -19,7 +22,7 @@ def get_accepted_lang(request): """ returns a list of language codes accepted by the HTTP request """ lang_str = request.META.get('HTTP_ACCEPT_LANGUAGE', '') - lang_str = [c for c in lang_str if c in string.ascii_letters+','] + lang_str = ''.join([c for c in lang_str if c in string.ascii_letters+',']) langs = lang_str.split(',') langs = [s[:2] for s in langs] langs = list(map(str.strip, langs)) @@ -42,8 +45,9 @@ def sanitize_language_codes(ls): >>> sanitize_language_codes(['de-at', 'de-ch']) ['de'] - >>> sanitize_language_codes(['de-at', 'en', 'en-gb', '(asdf', 'Deutsch']) - ['de', 'en'] + >>> sanitize_language_codes(['de-at', 'en', 'en-gb', '(asdf', 'Deutsch']) \ + == ['de', 'en'] + True """ ls = [sanitize_language_code(l) for l in ls if l and RE_LANG.match(l)] @@ -75,6 +79,9 @@ def get_page_list(start, total, cur, show_max): >>> get_page_list(1, 100, 1, 10) [1, 2, 3, 4, 5, 6, '...', 98, 99, 100] + >>> get_page_list(1, 995/10, 1, 10) + [1, 2, 3, 4, 5, 6, '...', 98, 99, 100] + >>> get_page_list(1, 100, 50, 10) [1, '...', 48, 49, 50, 51, '...', 98, 99, 100] @@ -85,14 +92,18 @@ def get_page_list(start, total, cur, show_max): [1, 2, 3] """ + # if we get "total" as a float (eg from total_entries / entries_per_page) + # we round up + total = math.ceil(total) + if show_max >= (total - start): return list(range(start, total+1)) ps = [] if (cur - start) > show_max / 2: - ps.extend(list(range(start, show_max / 4))) + ps.extend(list(range(start, int(show_max / 4)))) ps.append('...') - ps.extend(list(range(cur - show_max / 4, cur))) + ps.extend(list(range(cur - int(show_max / 4), cur))) else: ps.extend(list(range(start, cur))) @@ -101,10 +112,10 @@ def get_page_list(start, total, cur, show_max): if (total - cur) > show_max / 2: # for the first pages, show more pages at the beginning - add = show_max / 2 - len(ps) - ps.extend(list(range(cur + 1, cur + show_max / 4 + add))) + add = math.ceil(show_max / 2 - len(ps)) + ps.extend(list(range(cur + 1, cur + int(show_max / 4) + add))) ps.append('...') - ps.extend(list(range(total - show_max / 4, total + 1))) + ps.extend(list(range(total - int(show_max / 4), total + 1))) else: ps.extend(list(range(cur + 1, total + 1))) @@ -248,23 +259,23 @@ def hours_to_str(hours_total): """ returns a human-readable string representation of some hours >>> hours_to_str(1) - u'1 hour' + '1 hour' >>> hours_to_str(5) - u'5 hours' + '5 hours' >>> hours_to_str(100) - u'4 days, 4 hours' + '4 days, 4 hours' >>> hours_to_str(960) - u'5 weeks, 5 days' + '5 weeks, 5 days' >>> hours_to_str(961) - u'5 weeks, 5 days, 1 hour' + '5 weeks, 5 days, 1 hour' """ - weeks = hours_total / 24 / 7 - days = hours_total / 24 % 7 + weeks = int(hours_total / 24 / 7) + days = int(hours_total / 24) % 7 hours = hours_total % 24 strs = [] diff --git a/mygpo/web/views/podcast.py b/mygpo/web/views/podcast.py index 5c89a1e3..9f33a6fc 100644 --- a/mygpo/web/views/podcast.py +++ b/mygpo/web/views/podcast.py @@ -52,7 +52,8 @@ def show(request, podcast): episodes = episode_list(podcast, request.user, limit=num_episodes) user = request.user - max_listeners = max([e.listeners for e in episodes] + [0]) + listeners = list(filter(None, [e.listeners for e in episodes])) + max_listeners = max(listeners + [0]) episode = None diff --git a/requirements.txt b/requirements.txt index 32c8a6c8..88d8e27e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ feedparser==5.1.3 gunicorn==19.1.1 html2text==2014.7.3 markdown2==2.2.2 -oauth2client==1.2 +oauth2client==1.4.11 psycopg2==2.5.4 pyes==0.99.5 python-dateutil==2.2 -- 2.11.4.GIT