mygpo/maintenance/migrate.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 from datetime import datetime
   5
   6 from django.contrib.contenttypes.models import ContentType
   7 from django.contrib.auth.models import User
   8 from django.utils.text import slugify
   9 from django.db import reset_queries
  10
  11 from mygpo.podcasts.models import Tag
  12 from mygpo.users.models import EpisodeUserState, Client
  13 from mygpo.chapters.models import Chapter
  14 from mygpo.subscriptions.models import Subscription, PodcastConfig
  15 from mygpo.podcastlists.models import PodcastList, PodcastListEntry
  16 from mygpo.history.models import EpisodeHistoryEntry
  17 from mygpo.podcasts.models import Episode, Podcast, PodcastGroup
  18 from mygpo.directory.models import Category as C
  19 from mygpo.categories.models import Category, CategoryEntry, CategoryTag
  20 from mygpo.favorites.models import FavoriteEpisode
  21 from mygpo.votes.models import Vote
  22
  23 import logging
  24 logger = logging.getLogger(__name__)
  25
  26
  27 def to_maxlength(cls, field, val):
  28     """ Cut val to the maximum length of cls's field """
  29     max_length = cls._meta.get_field(field).max_length
  30     orig_length = len(val)
  31     if orig_length > max_length:
  32         val = val[:max_length]
  33         logger.warn('%s.%s length reduced from %d to %d',
  34                     cls.__name__, field, orig_length, max_length)
  35
  36     return val
  37
  38
  39 #class EpisodeUserState(Document, SettingsMixin):
  40 #    ref_url       = StringProperty(required=True)
  41 #    podcast_ref_url = StringProperty(required=True)
  42
  43
  44 def migrate_estate(state):
  45     """ migrate a podcast state """
  46
  47     try:
  48         user = User.objects.get(profile__uuid=state.user)
  49     except User.DoesNotExist:
  50         logger.warn("User with ID '{id}' does not exist".format(
  51             id=state.user))
  52         return
  53
  54     try:
  55         podcast = Podcast.objects.all().get_by_any_id(state.podcast)
  56     except Podcast.DoesNotExist:
  57         logger.warn("Podcast with ID '{id}' does not exist".format(
  58             id=state.podcast))
  59         return
  60
  61     try:
  62         episode = Episode.objects.filter(podcast=podcast).get_by_any_id(state.episode)
  63     except Episode.DoesNotExist:
  64         logger.warn("Episode with ID '{id}' does not exist".format(
  65             id=state.episode))
  66         return
  67
  68     logger.info('Migrating episode state ({id}) for user {user} and episode {episode}'
  69                 .format(id=state._id, user=user, episode=episode))
  70
  71     for action in state.actions:
  72         migrate_eaction(user, episode, state, action)
  73
  74
  75 def migrate_eaction(user, episode, state, ea):
  76
  77     logger.info('Migrating {action} action'.format(action=ea.action))
  78
  79     if ea.device is None:
  80         client = None
  81
  82     else:
  83         try:
  84             client = user.client_set.get(id=ea.device)
  85         except Client.DoesNotExist:
  86             logger.warn("Client '{cid}' does not exist; skipping".format(
  87                 cid=ea.device))
  88             return
  89
  90     created = datetime.utcfromtimestamp(ea.upload_timestamp) if ea.upload_timestamp else datetime.utcnow()
  91     entry, created = EpisodeHistoryEntry.objects.get_or_create(
  92         user=user,
  93         client=client,
  94         episode=episode,
  95         action=ea.action,
  96         timestamp=ea.timestamp,
  97         defaults = {
  98             'created': created,
  99             'started': ea.started,
 100             'stopped': ea.playmark,
 101             'total': ea.total,
 102             'podcast_ref_url': state.podcast_ref_url,
 103             'episode_ref_url': state.ref_url,
 104         },
 105     )
 106
 107     if created:
 108         logger.info('Episode History Entry created: {user} {action} {episode}'
 109                     'on {client} @ {timestamp}'.format(user=user,
 110                         action=entry.action, episode=episode, client=client,
 111                         timestamp=entry.timestamp))
 112
 113
 114 def migrate_category(cat):
 115
 116     logger.info('Migrating category {category}'.format(category=cat))
 117     category, created = Category.objects.get_or_create(
 118         title=to_maxlength(Category, 'title', cat.label)
 119     )
 120
 121     for spelling in cat.spellings + [cat.label]:
 122         s, c = CategoryTag.objects.get_or_create(
 123             tag=slugify(to_maxlength(CategoryTag, 'tag', spelling.strip())),
 124             defaults={
 125                 'category': category,
 126             }
 127         )
 128
 129     for podcast_id in cat.podcasts:
 130         if isinstance(podcast_id, dict):
 131             podcast_id = podcast_id['podcast']
 132         logger.info(repr(podcast_id))
 133
 134         try:
 135             podcast = Podcast.objects.all().get_by_any_id(podcast_id)
 136         except Podcast.DoesNotExist:
 137             logger.warn("Podcast with ID '{podcast_id}' does not exist".format(
 138                 podcast_id=podcast_id))
 139             continue
 140
 141         entry, c = CategoryEntry.objects.get_or_create(category=category,
 142                                                        podcast=podcast)
 143
 144     category.save()
 145
 146
 147 from couchdbkit import Database
 148 db = Database('http://127.0.0.1:5984/mygpo_userdata_copy')
 149 from couchdbkit.changes import ChangesStream, fold, foreach
 150
 151
 152 MIGRATIONS = {
 153     'PodcastUserState': (None, None),
 154     'User': (None, None),
 155     'Suggestions': (None, None),
 156     'EpisodeUserState': (EpisodeUserState, migrate_estate),
 157     'PodcastList': (None, None),
 158     'Category': (C, migrate_category),
 159 }
 160
 161 def migrate_change(c):
 162     logger.info('Migrate seq %s', c['seq'])
 163     doc = c['doc']
 164
 165     if not 'doc_type' in doc:
 166         logger.warn('Document contains no doc_type: %r', doc)
 167         return
 168
 169     doctype = doc['doc_type']
 170
 171     cls, migrate = MIGRATIONS[doctype]
 172
 173     if cls is None:
 174         logger.warn("Skipping '%s'", doctype)
 175         return
 176
 177     obj = cls.wrap(doc)
 178     migrate(obj)
 179     reset_queries()
 180
 181
 182 def migrate(since=0, db=db):
 183     with ChangesStream(db,
 184                        feed="continuous",
 185                        heartbeat=True,
 186                        include_docs=True,
 187                        since=since,
 188                     ) as stream:
 189         for change in stream:
 190             migrate_change(change)