stgit/lib/log.py

   1 """Functions for managing stack state log.
   2
   3 A stack log is a Git branch. Each commit contains the complete state (metadata) of the
   4 stack at the moment it was written; the most recent commit has the most recent state.
   5
   6 For a branch `foo`, the stack state log is stored in the ref `refs/stacks/foo`.
   7
   8 Each log entry makes sure to have proper references to everything it needs to make it
   9 safe against garbage collection. The stack state log can even be pulled from one
  10 repository to another.
  11
  12 Format version 0
  13 ================
  14
  15 Version 0 was an experimental version of the stack log format; it is no longer
  16 supported.
  17
  18 Format version 1
  19 ================
  20
  21 Commit message
  22 --------------
  23
  24 The commit message is mostly for human consumption; in most cases it is just a subject
  25 line: the StGit subcommand name and possibly some important command-line flag.
  26
  27 An exception to this is commits for ``undo`` and ``redo``. Their subject line is "undo
  28 `n`" and "redo `n`"; the positive integer `n` indicates how many steps were undone or
  29 redone.
  30
  31 Tree
  32 ----
  33
  34 * One blob, ``meta``, that contains the stack state:
  35
  36    * ``Version: 1``
  37
  38      Future versions of StGit might change the log format; when this is done, this
  39      version number will be incremented.
  40
  41    * ``Previous: <sha1 or None>``
  42
  43      The commit of the previous log entry, or None if this is the first entry.
  44
  45    * ``Head: <sha1>``
  46
  47      The current branch head.
  48
  49    * ``Applied:``
  50
  51      Marks the start of the list of applied patches. Each patch is listed in order, one
  52      per line. First one or more spaces, then the patch name, then a colon, space, then
  53      the patch's sha1.
  54
  55    * ``Unapplied:``
  56
  57      Same as ``Applied:``, but for the unapplied patches.
  58
  59    * ``Hidden:``
  60
  61      Same as ``Applied:``, but for the hidden patches.
  62
  63 * One subtree, ``patches``, that contains one blob per patch::
  64
  65    Bottom: <sha1 of patch's bottom tree>
  66    Top:    <sha1 of patch's top tree>
  67    Author: <author name and e-mail>
  68    Date:   <patch timestamp>
  69
  70    <commit message>
  71
  72 Parents
  73 -------
  74
  75 * The first parent is the *simplified log*, described below.
  76
  77 * The rest of the parents are just there to make sure that all the commits referred to
  78   in the log entry--patches, branch head, previous log entry--are ancestors of the log
  79   commit. This is necessary to make the log safe with regard to garbage collection and
  80   pulling.
  81
  82 Simplified log
  83 --------------
  84
  85 The simplified log is exactly like the full log, except that its only parent is the
  86 (simplified) previous log entry, if any. It's purpose is mainly ease of visualization
  87 in, for example, ``git log --graph`` or ``gitk``.
  88
  89 Format version 4
  90 ================
  91
  92 The metadata in `refs/heads/<branch>.stgit` branch is the same as format version 1.
  93
  94 Format version 4 indicates that, unlike previous format versions used by older versions
  95 of StGit, the stack log state is *only* contained in the stack log branch and *not* as
  96 files in the .git/patches directory.
  97
  98 Format version 5
  99 ================
 100
 101 Stack metadata resides in `refs/stacks/<branch>` where the `stack.json` metadata file
 102 is JSON format instead of the previous custom format.
 103
 104 """
 105
 106 import json
 107 import re
 108
 109 from stgit.exception import StgException
 110 from stgit.lib.git import BlobData, CommitData, TreeData
 111 from stgit.lib.stackupgrade import FORMAT_VERSION
 112 from stgit.out import out
 113
 114
 115 class LogException(StgException):
 116     pass
 117
 118
 119 class LogParseException(LogException):
 120     pass
 121
 122
 123 class StackState:
 124     _max_parents = 16
 125
 126     def __init__(
 127         self,
 128         prev,
 129         head,
 130         applied,
 131         unapplied,
 132         hidden,
 133         patches,
 134         commit=None,
 135     ):
 136         self.prev = prev
 137         self.head = head
 138         self.applied = applied
 139         self.unapplied = unapplied
 140         self.hidden = hidden
 141         self.patches = patches
 142         self.commit = commit
 143
 144     @property
 145     def simplified_parent(self):
 146         return self.commit.data.parents[0]
 147
 148     def get_prev_state(self, repo):
 149         if self.prev is None:
 150             return None
 151         else:
 152             return self.from_commit(repo, self.prev)
 153
 154     @property
 155     def base(self):
 156         if self.applied:
 157             return self.patches[self.applied[0]].data.parent
 158         else:
 159             return self.head
 160
 161     @property
 162     def top(self):
 163         if self.applied:
 164             return self.patches[self.applied[-1]]
 165         else:
 166             return self.head
 167
 168     @property
 169     def all_patches(self):
 170         return self.applied + self.unapplied + self.hidden
 171
 172     @classmethod
 173     def new_empty(cls, head):
 174         return cls(
 175             prev=None,
 176             head=head,
 177             applied=[],
 178             unapplied=[],
 179             hidden=[],
 180             patches={},
 181         )
 182
 183     @classmethod
 184     def from_stack(cls, prev, stack):
 185         return cls(
 186             prev=prev,
 187             head=stack.head,
 188             applied=list(stack.patchorder.applied),
 189             unapplied=list(stack.patchorder.unapplied),
 190             hidden=list(stack.patchorder.hidden),
 191             patches={pn: stack.patches[pn] for pn in stack.patchorder.all},
 192         )
 193
 194     @classmethod
 195     def from_commit(cls, repo, commit):
 196         """Parse a (full or simplified) stack log commit."""
 197         try:
 198             perm, stack_json_blob = commit.data.tree.data['stack.json']
 199         except KeyError:
 200             raise LogParseException('Not a stack log')
 201
 202         try:
 203             stack_json = json.loads(stack_json_blob.data.bytes.decode('utf-8'))
 204         except json.JSONDecodeError as e:
 205             raise LogParseException(str(e))
 206
 207         version = stack_json.get('version')
 208
 209         if version is None:
 210             raise LogException('Missing stack metadata version')
 211         elif version < FORMAT_VERSION:
 212             raise LogException('Log is version %d, which is too old' % version)
 213         elif version > FORMAT_VERSION:
 214             raise LogException('Log is version %d, which is too new' % version)
 215
 216         patches = {
 217             pn: repo.get_commit(patch_info['oid'])
 218             for pn, patch_info in stack_json['patches'].items()
 219         }
 220
 221         if stack_json['prev'] is None:
 222             prev = None
 223         else:
 224             prev = repo.get_commit(stack_json['prev'])
 225
 226         return cls(
 227             prev,
 228             repo.get_commit(stack_json['head']),
 229             stack_json['applied'],
 230             stack_json['unapplied'],
 231             stack_json['hidden'],
 232             patches,
 233             commit,
 234         )
 235
 236     def _parents(self, prev_state):
 237         """Parents this entry needs to be a descendant of all commits it refers to."""
 238         xp = {self.head, self.top}
 239         xp |= {self.patches[pn] for pn in self.unapplied}
 240         xp |= {self.patches[pn] for pn in self.hidden}
 241         if prev_state is not None:
 242             xp.add(prev_state.commit)
 243             xp -= set(prev_state.patches.values())
 244         return xp
 245
 246     def _stack_json_blob(self, repo, prev_state):
 247         stack_json = dict(
 248             version=FORMAT_VERSION,
 249             prev=None if prev_state is None else prev_state.commit.sha1,
 250             head=self.head.sha1,
 251             applied=self.applied,
 252             unapplied=self.unapplied,
 253             hidden=self.hidden,
 254             patches={pn: dict(oid=patch.sha1) for pn, patch in self.patches.items()},
 255         )
 256         blob = json.dumps(stack_json, indent=2).encode('utf-8')
 257         return repo.commit(BlobData(blob))
 258
 259     def _patch_blob(self, repo, pn, commit, prev_state):
 260         if prev_state is not None:
 261             perm, prev_patch_tree = prev_state.commit.data.tree.data['patches']
 262             if pn in prev_patch_tree.data and commit == prev_state.patches[pn]:
 263                 return prev_patch_tree.data[pn]
 264
 265         patch_meta = '\n'.join(
 266             [
 267                 'Bottom: %s' % commit.data.parent.data.tree.sha1,
 268                 'Top:    %s' % commit.data.tree.sha1,
 269                 'Author: %s' % commit.data.author.name_email,
 270                 'Date:   %s' % commit.data.author.date,
 271                 '',
 272                 commit.data.message_str,
 273             ]
 274         )
 275         return repo.commit(BlobData(patch_meta.encode('utf-8')))
 276
 277     def _patches_tree(self, repo, prev_state):
 278         return repo.commit(
 279             TreeData(
 280                 {
 281                     pn: self._patch_blob(repo, pn, commit, prev_state)
 282                     for pn, commit in self.patches.items()
 283                 }
 284             )
 285         )
 286
 287     def _tree(self, repo, prev_state):
 288         return repo.commit(
 289             TreeData(
 290                 {
 291                     'stack.json': self._stack_json_blob(repo, prev_state),
 292                     'patches': self._patches_tree(repo, prev_state),
 293                 }
 294             )
 295         )
 296
 297     def commit_state(self, repo, message):
 298         """Commit stack state to stack metadata branch."""
 299         prev_state = self.get_prev_state(repo)
 300         tree = self._tree(repo, prev_state)
 301         simplified_parent = repo.commit(
 302             CommitData(
 303                 tree=tree,
 304                 message=message,
 305                 parents=[] if prev_state is None else [prev_state.simplified_parent],
 306             )
 307         )
 308         parents = list(self._parents(prev_state))
 309         while len(parents) >= self._max_parents:
 310             g = repo.commit(
 311                 CommitData(
 312                     tree=tree,
 313                     parents=parents[-self._max_parents :],
 314                     message='Stack log parent grouping',
 315                 )
 316             )
 317             parents[-self._max_parents :] = [g]
 318         self.commit = repo.commit(
 319             CommitData(
 320                 tree=tree,
 321                 message=message,
 322                 parents=[simplified_parent] + parents,
 323             )
 324         )
 325         return self.commit
 326
 327     def same_state(self, other):
 328         """Check whether two stack state entries describe the same stack state."""
 329         return (
 330             self.head == other.head
 331             and self.applied == other.applied
 332             and self.unapplied == other.unapplied
 333             and self.hidden == other.hidden
 334             and self.patches == other.patches
 335         )
 336
 337
 338 def get_stack_state(repo, ref, commit=None):
 339     if commit is None:
 340         commit = repo.refs.get(ref)  # May raise KeyError
 341     try:
 342         return StackState.from_commit(repo, commit)
 343     except LogException as e:
 344         raise LogException('While reading log from %s: %s' % (ref, e))
 345
 346
 347 def log_stack_state(stack, msg):
 348     """Write a new metadata entry for the stack."""
 349     try:
 350         prev_state = get_stack_state(stack.repository, stack.state_ref)
 351     except KeyError:
 352         prev_state = None
 353
 354     try:
 355         new_state = StackState.from_stack(prev_state.commit, stack)
 356     except LogException as e:
 357         out.warn(str(e), 'No log entry written.')
 358     else:
 359         if not prev_state or not new_state.same_state(prev_state):
 360             state_commit = new_state.commit_state(stack.repository, msg)
 361             stack.repository.refs.set(stack.state_ref, state_commit, msg)
 362
 363
 364 def reset_stack(trans, iw, state):
 365     """Reset the stack to a given previous state."""
 366     for pn in trans.all_patches:
 367         trans.patches[pn] = None
 368     for pn in state.all_patches:
 369         trans.patches[pn] = state.patches[pn]
 370     trans.applied = state.applied
 371     trans.unapplied = state.unapplied
 372     trans.hidden = state.hidden
 373     trans.base = state.base
 374     trans.head = state.head
 375
 376
 377 def reset_stack_partially(trans, iw, state, only_patches):
 378     """Reset the stack to a previous state, but only for the given patches.
 379
 380     :param only_patches: Touch only these patches
 381     :type only_patches: iterable
 382
 383     """
 384     only_patches = set(only_patches)
 385     patches_to_reset = set(state.all_patches) & only_patches
 386     existing_patches = set(trans.all_patches)
 387     original_applied_order = list(trans.applied)
 388     to_delete = (existing_patches - patches_to_reset) & only_patches
 389
 390     # In one go, do all the popping we have to in order to pop the
 391     # patches we're going to delete or modify.
 392     def mod(pn):
 393         if pn not in only_patches:
 394             return False
 395         if pn in to_delete:
 396             return True
 397         if trans.patches[pn] != state.patches.get(pn, None):
 398             return True
 399         return False
 400
 401     trans.pop_patches(mod)
 402
 403     # Delete and modify/create patches. We've previously popped all
 404     # patches that we touch in this step.
 405     trans.delete_patches(lambda pn: pn in to_delete)
 406     for pn in patches_to_reset:
 407         if pn in existing_patches:
 408             if trans.patches[pn] == state.patches[pn]:
 409                 continue
 410             else:
 411                 out.info('Resetting %s' % pn)
 412         else:
 413             if pn in state.hidden:
 414                 trans.hidden.append(pn)
 415             else:
 416                 trans.unapplied.append(pn)
 417             out.info('Resurrecting %s' % pn)
 418         trans.patches[pn] = state.patches[pn]
 419
 420     # Push all the patches that we've popped, if they still
 421     # exist.
 422     pushable = set(trans.unapplied + trans.hidden)
 423     for pn in original_applied_order:
 424         if pn in pushable:
 425             trans.push_patch(pn, iw)
 426
 427
 428 def undo_state(stack, undo_steps):
 429     """Find the stack state ``undo_steps`` steps in the past.
 430
 431     Successive undo operations are supposed to "add up", so if we find other undo
 432     operations along the way we have to add those undo steps to ``undo_steps``.
 433
 434     If ``undo_steps`` is negative, redo instead of undo.
 435
 436     :returns: stack state that is the destination of the undo operation
 437     :rtype: :class:`StackState`
 438
 439     """
 440     try:
 441         state = get_stack_state(stack.repository, stack.state_ref)
 442     except KeyError:
 443         raise LogException('Log is empty')
 444     while undo_steps != 0:
 445         msg = state.commit.data.message_str.strip()
 446         um = re.match(r'^undo\s+(\d+)$', msg)
 447         if undo_steps > 0:
 448             if um:
 449                 undo_steps += int(um.group(1))
 450             else:
 451                 undo_steps -= 1
 452         else:
 453             rm = re.match(r'^redo\s+(\d+)$', msg)
 454             if um:
 455                 undo_steps += 1
 456             elif rm:
 457                 undo_steps -= int(rm.group(1))
 458             else:
 459                 raise LogException('No more redo information available')
 460         prev_state = state.get_prev_state(stack.repository)
 461         if prev_state is None:
 462             raise LogException('Not enough undo information available')
 463         state = prev_state
 464     return state
 465
 466
 467 def log_external_mods(stack):
 468     try:
 469         state = get_stack_state(stack.repository, stack.state_ref)
 470     except KeyError:
 471         # No log exists yet.
 472         log_stack_state(stack, 'start of log')
 473         return
 474     except LogException:
 475         # Something's wrong with the log, so don't bother.
 476         return
 477     if state.head == stack.head:
 478         # No external modifications.
 479         return
 480     log_stack_state(
 481         stack,
 482         'external modifications\n\n'
 483         'Modifications by tools other than StGit (e.g. git).\n',
 484     )