mygpo/maintenance/merge.py

   1 from itertools import chain, imap as map
   2 import logging
   3 from functools import partial
   4
   5 import restkit
   6
   7 from mygpo import utils
   8 from mygpo.decorators import repeat_on_conflict
   9 from mygpo.db.couchdb.episode import episodes_for_podcast
  10 from mygpo.db.couchdb.podcast_state import all_podcast_states
  11 from mygpo.db.couchdb.episode_state import all_episode_states
  12
  13
  14 class IncorrectMergeException(Exception):
  15     pass
  16
  17
  18 class PodcastMerger(object):
  19     """ Merges podcast2 into podcast
  20
  21     Also merges the related podcast states, and re-assignes podcast2's episodes
  22     to podcast, but does neither merge their episodes nor their episode states
  23     """
  24
  25
  26     def __init__(self, podcasts, actions, groups):
  27
  28         for n, podcast1 in enumerate(podcasts):
  29             for m, podcast2 in enumerate(podcasts):
  30                 if podcast1 == podcast2 and n != m:
  31                     raise IncorrectMergeException("can't merge podcast into itself")
  32
  33         self.podcasts = podcasts
  34         self.actions = actions
  35         self.groups = groups
  36
  37
  38     def merge(self):
  39         podcast1 = self.podcasts.pop(0)
  40
  41         for podcast2 in self.podcasts:
  42             self._merge_objs(podcast1=podcast1, podcast2=podcast2)
  43             self.merge_states(podcast1, podcast2)
  44             self.merge_episodes()
  45             self.reassign_episodes(podcast1, podcast2)
  46             self._delete(podcast2=podcast2)
  47
  48         self.actions['merge-podcast'] += 1
  49
  50
  51     def merge_episodes(self):
  52         for n, episodes in self.groups:
  53
  54             episode = episodes.pop(0)
  55
  56             for ep in episodes:
  57
  58                 em = EpisodeMerger(episode, ep, self.actions)
  59                 em.merge()
  60
  61
  62     @repeat_on_conflict(['podcast1', 'podcast2'])
  63     def _merge_objs(self, podcast1, podcast2):
  64
  65         podcast1.merged_ids = set_filter(podcast1.get_id(),
  66                 podcast1.merged_ids, [podcast2.get_id()], podcast2.merged_ids)
  67
  68         podcast1.merged_slugs = set_filter(podcast1.slug,
  69                 podcast1.merged_slugs, [podcast2.slug], podcast2.merged_slugs)
  70
  71         podcast1.merged_oldids = set_filter(podcast1.oldid,
  72                 podcast1.merged_oldids, [podcast2.oldid],
  73                 podcast2.merged_oldids)
  74
  75         # the first URL in the list represents the podcast main URL
  76         main_url = podcast1.url
  77         podcast1.urls = set_filter(None, podcast1.urls, podcast2.urls)
  78         # so we insert it as the first again
  79         podcast1.urls.remove(main_url)
  80         podcast1.urls.insert(0, main_url)
  81
  82         # we ignore related_podcasts because
  83         # * the elements should be roughly the same
  84         # * element order is important but could not preserved exactly
  85
  86         podcast1.content_types = set_filter(None, podcast1.content_types,
  87                 podcast2.content_types)
  88
  89         key = lambda x: x.timestamp
  90         for a, b in utils.iterate_together(
  91                 [podcast1.subscribers, podcast2.subscribers], key):
  92
  93             if a is None or b is None:
  94                 continue
  95
  96             # avoid increasing subscriber_count when merging
  97             # duplicate entries of a single podcast
  98             if a.subscriber_count == b.subscriber_count:
  99                 continue
 100
 101             a.subscriber_count += b.subscriber_count
 102
 103         for src, tags in podcast2.tags.items():
 104             podcast1.tags[src] = set_filter(None, podcast1.tags.get(src, []),
 105                     tags)
 106
 107         podcast1.save()
 108
 109
 110     @repeat_on_conflict(['podcast2'])
 111     def _delete(self, podcast2):
 112         podcast2.delete()
 113
 114
 115     @repeat_on_conflict(['s'])
 116     def _save_state(self, s, podcast1):
 117         s.podcast = podcast1.get_id()
 118         s.save()
 119
 120
 121     @repeat_on_conflict(['e'])
 122     def _save_episode(self, e, podcast1):
 123         e.podcast = podcast1.get_id()
 124         e.save()
 125
 126
 127
 128     def reassign_episodes(self, podcast1, podcast2):
 129         # re-assign episodes to new podcast
 130         # if necessary, they will be merged later anyway
 131         for e in episodes_for_podcast(podcast2):
 132             self.actions['reassign-episode'] += 1
 133
 134             for s in all_episode_states(e):
 135                 self.actions['reassign-episode-state'] += 1
 136
 137                 self._save_state(s=s, podcast1=podcast1)
 138
 139             self._save_episode(e=e, podcast1=podcast1)
 140
 141
 142     def merge_states(self, podcast1, podcast2):
 143         """Merges the Podcast states that are associated with the two Podcasts.
 144
 145         This should be done after two podcasts are merged
 146         """
 147
 148         key = lambda x: x.user
 149         states1 = sorted(all_podcast_states(podcast1), key=key)
 150         states2 = sorted(all_podcast_states(podcast2), key=key)
 151
 152         for state, state2 in utils.iterate_together([states1, states2], key):
 153
 154             if state == state2:
 155                 continue
 156
 157             if state is None:
 158                 self.actions['move-podcast-state'] += 1
 159                 self._move_state(state2=state2, new_id=podcast1.get_id(),
 160                         new_url=podcast1.url)
 161
 162             elif state2 is None:
 163                 continue
 164
 165             else:
 166                 psm = PodcastStateMerger(state, state2, self.actions)
 167                 psm.merge()
 168
 169
 170     @repeat_on_conflict(['state2'])
 171     def _move_state(self, state2, new_id, new_url):
 172         state2.ref_url = new_url
 173         state2.podcast = new_id
 174         state2.save()
 175
 176     @repeat_on_conflict(['state2'])
 177     def _delete_state(state2):
 178         state2.delete()
 179
 180
 181
 182 class EpisodeMerger(object):
 183
 184
 185     def __init__(self, episode1, episode2, actions):
 186         if episode1 == episode2:
 187             raise IncorrectMergeException("can't merge episode into itself")
 188
 189         self.episode1 = episode1
 190         self.episode2 = episode2
 191         self.actions = actions
 192
 193
 194     def merge(self):
 195         self._merge_objs(episode1=self.episode1, episode2=self.episode2)
 196         self.merge_states(self.episode1, self.episode2)
 197         self._delete(e=self.episode2)
 198         self.actions['merge-episode'] += 1
 199
 200
 201     @repeat_on_conflict(['episode1'])
 202     def _merge_objs(self, episode1, episode2):
 203
 204         episode1.urls = set_filter(None, episode1.urls, episode2.urls)
 205
 206         episode1.merged_ids = set_filter(episode1._id, episode1.merged_ids,
 207                 [episode2._id], episode2.merged_ids)
 208
 209         episode1.merged_slugs = set_filter(episode1.slug,
 210                 episode1.merged_slugs, [episode2.slug], episode2.merged_slugs)
 211
 212         episode1.save()
 213
 214
 215     @repeat_on_conflict(['e'])
 216     def _delete(self, e):
 217         e.delete()
 218
 219
 220     def merge_states(self, episode, episode2):
 221
 222         key = lambda x: x.user
 223         states1 = sorted(all_episode_states(self.episode1), key=key)
 224         states2 = sorted(all_episode_states(self.episode2), key=key)
 225
 226         for state, state2 in utils.iterate_together([states1, states2], key):
 227
 228             if state == state2:
 229                 continue
 230
 231             if state is None:
 232                 self.actions['move-episode-state'] += 1
 233                 self._move(state2=state2, podcast_id=self.episode1.podcast,
 234                         episode_id=self.episode1._id)
 235
 236             elif state2 is None:
 237                 continue
 238
 239             else:
 240                 esm = EpisodeStateMerger(state, state2, self.actions)
 241                 esm.merge()
 242
 243
 244     @repeat_on_conflict(['state2'])
 245     def _move(self, state2, podcast_id, episode_id):
 246         state2.podcast = podcast_id
 247         state2.episode = episode_id
 248         state2.save()
 249
 250
 251
 252
 253
 254
 255 class PodcastStateMerger(object):
 256     """Merges the two given podcast states"""
 257
 258     def __init__(self, state, state2, actions):
 259
 260         if state._id == state2._id:
 261             raise IncorrectMergeException("can't merge podcast state into itself")
 262
 263         if state.user != state2.user:
 264             raise IncorrectMergeException("states don't belong to the same user")
 265
 266         self.state = state
 267         self.state2 = state2
 268         self.actions = actions
 269
 270
 271     def merge(self):
 272         self._do_merge(state=self.state, state2=self.state2)
 273         self._add_actions(state=self.state, actions=self.state2.actions)
 274         self._delete(state2=self.state2)
 275         self.actions['merged-podcast-state'] += 1
 276
 277
 278     @repeat_on_conflict(['state'])
 279     def _do_merge(self, state, state2):
 280
 281         # overwrite settings in state2 with state's settings
 282         settings = state2.settings
 283         settings.update(state.settings)
 284         state.settings = settings
 285
 286         state.disabled_devices = set_filter(None, state.disabled_devices,
 287                 state2.disabled_devices)
 288
 289         state.merged_ids = set_filter(state1._id, state.merged_ids,
 290                 [state2._id], state2.merged_ids)
 291
 292         state.tags = set_filter(None, state.tags, state2.tags)
 293
 294         state.save()
 295
 296
 297     @repeat_on_conflict(['state'])
 298     def _add_actions(self, state, actions):
 299         try:
 300             state.add_actions(actions)
 301             state.save()
 302         except restkit.Unauthorized:
 303             # the merge could result in an invalid list of
 304             # subscribe/unsubscribe actions -- we ignore it and
 305             # just use the actions from state
 306             return
 307
 308     @repeat_on_conflict(['state2'])
 309     def _delete(self, state2):
 310         state2.delete()
 311
 312
 313
 314
 315
 316 class EpisodeStateMerger(object):
 317     """ Merges state2 in state """
 318
 319     def __init__(self, state, state2, actions):
 320
 321         if state._id == state2._id:
 322             raise IncorrectMergeException("can't merge episode state into itself")
 323
 324         if state.user != state2.user:
 325             raise IncorrectMergeException("states don't belong to the same user")
 326
 327         self.state = state
 328         self.state2 = state2
 329         self.actions = actions
 330
 331
 332     def merge(self):
 333         self._merge_obj(state=self.state, state2=self.state2)
 334         self._do_delete(state2=self.state2)
 335         self.actions['merge-episode-state'] += 1
 336
 337
 338     @repeat_on_conflict(['state'])
 339     def _merge_obj(self, state, state2):
 340         state.add_actions(state2.actions)
 341
 342         # overwrite settings in state2 with state's settings
 343         settings = state2.settings
 344         settings.update(state.settings)
 345         state.settings = settings
 346
 347         merged_ids = set(state.merged_ids + [state2._id] + state2.merged_ids)
 348         state.merged_ids = filter(None, merged_ids)
 349
 350         state.chapters = list(set(state.chapters + state2.chapters))
 351
 352         state.save()
 353
 354     @repeat_on_conflict(['state2'])
 355     def _do_delete(self, state2):
 356         state2.delete()
 357
 358
 359 def set_filter(orig, *args):
 360     """ chain args, and remove falsy values and orig """
 361     s = set(chain.from_iterable(args))
 362     s = s - set([orig])
 363     s = filter(None, s)
 364     return s