cvs2svn_lib/symbol_strategy.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2008 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """SymbolStrategy classes determine how to convert symbols."""
  18
  19 import re
  20
  21 from cvs2svn_lib.common import FatalError
  22 from cvs2svn_lib.common import path_join
  23 from cvs2svn_lib.common import normalize_svn_path
  24 from cvs2svn_lib.log import Log
  25 from cvs2svn_lib.symbol import Trunk
  26 from cvs2svn_lib.symbol import TypedSymbol
  27 from cvs2svn_lib.symbol import Branch
  28 from cvs2svn_lib.symbol import Tag
  29 from cvs2svn_lib.symbol import ExcludedSymbol
  30 from cvs2svn_lib.symbol_statistics import SymbolPlanError
  31
  32
  33 class StrategyRule:
  34   """A single rule that might determine how to convert a symbol."""
  35
  36   def start(self, symbol_statistics):
  37     """This method is called once before get_symbol() is ever called.
  38
  39     The StrategyRule can override this method to do whatever it wants
  40     to prepare itself for work.  SYMBOL_STATISTICS is an instance of
  41     SymbolStatistics containing the statistics for all symbols in all
  42     projects."""
  43
  44     pass
  45
  46   def get_symbol(self, symbol, stats):
  47     """Return an object describing what to do with the symbol in STATS.
  48
  49     SYMBOL holds a Trunk or Symbol object as it has been determined so
  50     far.  Hopefully one of these method calls will turn any naked
  51     Symbol instances into TypedSymbols.
  52
  53     If this rule applies to the SYMBOL (whose statistics are collected
  54     in STATS), then return a new or modified AbstractSymbol object.
  55     If this rule doesn't apply, return SYMBOL unchanged."""
  56
  57     raise NotImplementedError()
  58
  59   def finish(self):
  60     """This method is called once after get_symbol() is done being called.
  61
  62     The StrategyRule can override this method do whatever it wants to
  63     release resources, etc."""
  64
  65     pass
  66
  67
  68 class _RegexpStrategyRule(StrategyRule):
  69   """A Strategy rule that bases its decisions on regexp matches.
  70
  71   If self.regexp matches a symbol name, return self.action(symbol);
  72   otherwise, return the symbol unchanged."""
  73
  74   def __init__(self, pattern, action):
  75     """Initialize a _RegexpStrategyRule.
  76
  77     PATTERN is a string that will be treated as a regexp pattern.
  78     PATTERN must match a full symbol name for the rule to apply (i.e.,
  79     it is anchored at the beginning and end of the symbol name).
  80
  81     ACTION is the class representing how the symbol should be
  82     converted.  It should be one of the classes Branch, Tag, or
  83     ExcludedSymbol.
  84
  85     If PATTERN matches a symbol name, then get_symbol() returns
  86     ACTION(name, id); otherwise it returns SYMBOL unchanged."""
  87
  88     try:
  89       self.regexp = re.compile('^' + pattern + '$')
  90     except re.error:
  91       raise FatalError("%r is not a valid regexp." % (pattern,))
  92
  93     self.action = action
  94
  95   def log(self, symbol):
  96     raise NotImplementedError()
  97
  98   def get_symbol(self, symbol, stats):
  99     if isinstance(symbol, (Trunk, TypedSymbol)):
 100       return symbol
 101     elif self.regexp.match(symbol.name):
 102       self.log(symbol)
 103       return self.action(symbol)
 104     else:
 105       return symbol
 106
 107
 108 class ForceBranchRegexpStrategyRule(_RegexpStrategyRule):
 109   """Force symbols matching pattern to be branches."""
 110
 111   def __init__(self, pattern):
 112     _RegexpStrategyRule.__init__(self, pattern, Branch)
 113
 114   def log(self, symbol):
 115     Log().verbose(
 116         'Converting symbol %s as a branch because it matches regexp "%s".'
 117         % (symbol, self.regexp.pattern,)
 118         )
 119
 120
 121 class ForceTagRegexpStrategyRule(_RegexpStrategyRule):
 122   """Force symbols matching pattern to be tags."""
 123
 124   def __init__(self, pattern):
 125     _RegexpStrategyRule.__init__(self, pattern, Tag)
 126
 127   def log(self, symbol):
 128     Log().verbose(
 129         'Converting symbol %s as a tag because it matches regexp "%s".'
 130         % (symbol, self.regexp.pattern,)
 131         )
 132
 133
 134 class ExcludeRegexpStrategyRule(_RegexpStrategyRule):
 135   """Exclude symbols matching pattern."""
 136
 137   def __init__(self, pattern):
 138     _RegexpStrategyRule.__init__(self, pattern, ExcludedSymbol)
 139
 140   def log(self, symbol):
 141     Log().verbose(
 142         'Excluding symbol %s because it matches regexp "%s".'
 143         % (symbol, self.regexp.pattern,)
 144         )
 145
 146
 147 class ExcludeTrivialImportBranchRule(StrategyRule):
 148   """If a symbol is a trivial import branch, exclude it.
 149
 150   A trivial import branch is defined to be a branch that only had a
 151   single import on it (no other kinds of commits) in every file in
 152   which it appeared.  In most cases these branches are worthless."""
 153
 154   def get_symbol(self, symbol, stats):
 155     if isinstance(symbol, (Trunk, TypedSymbol)):
 156       return symbol
 157     if stats.tag_create_count == 0 \
 158           and stats.branch_create_count == stats.trivial_import_count:
 159       Log().verbose(
 160           'Excluding branch %s because it is a trivial import branch.'
 161           % (symbol,)
 162           )
 163       return ExcludedSymbol(symbol)
 164     else:
 165       return symbol
 166
 167
 168 class ExcludeVendorBranchRule(StrategyRule):
 169   """If a symbol is a pure vendor branch, exclude it.
 170
 171   A pure vendor branch is defined to be a branch that only had imports
 172   on it (no other kinds of commits) in every file in which it
 173   appeared."""
 174
 175   def get_symbol(self, symbol, stats):
 176     if isinstance(symbol, (Trunk, TypedSymbol)):
 177       return symbol
 178     if stats.tag_create_count == 0 \
 179           and stats.branch_create_count == stats.pure_ntdb_count:
 180       Log().verbose(
 181           'Excluding branch %s because it is a pure vendor branch.'
 182           % (symbol,)
 183           )
 184       return ExcludedSymbol(symbol)
 185     else:
 186       return symbol
 187
 188
 189 class UnambiguousUsageRule(StrategyRule):
 190   """If a symbol is used unambiguously as a tag/branch, convert it as such."""
 191
 192   def get_symbol(self, symbol, stats):
 193     if isinstance(symbol, (Trunk, TypedSymbol)):
 194       return symbol
 195     is_tag = stats.tag_create_count > 0
 196     is_branch = stats.branch_create_count > 0 or stats.branch_commit_count > 0
 197     if is_tag and is_branch:
 198       # Can't decide
 199       return symbol
 200     elif is_branch:
 201       Log().verbose(
 202           'Converting symbol %s as a branch because it is always used '
 203           'as a branch.'
 204           % (symbol,)
 205           )
 206       return Branch(symbol)
 207     elif is_tag:
 208       Log().verbose(
 209           'Converting symbol %s as a tag because it is always used '
 210           'as a tag.'
 211           % (symbol,)
 212           )
 213       return Tag(symbol)
 214     else:
 215       # The symbol didn't appear at all:
 216       return symbol
 217
 218
 219 class BranchIfCommitsRule(StrategyRule):
 220   """If there was ever a commit on the symbol, convert it as a branch."""
 221
 222   def get_symbol(self, symbol, stats):
 223     if isinstance(symbol, (Trunk, TypedSymbol)):
 224       return symbol
 225     elif stats.branch_commit_count > 0:
 226       Log().verbose(
 227           'Converting symbol %s as a branch because there are commits on it.'
 228           % (symbol,)
 229           )
 230       return Branch(symbol)
 231     else:
 232       return symbol
 233
 234
 235 class HeuristicStrategyRule(StrategyRule):
 236   """Convert symbol based on how often it was used as a branch/tag.
 237
 238   Whichever happened more often determines how the symbol is
 239   converted."""
 240
 241   def get_symbol(self, symbol, stats):
 242     if isinstance(symbol, (Trunk, TypedSymbol)):
 243       return symbol
 244     elif stats.tag_create_count >= stats.branch_create_count:
 245       Log().verbose(
 246           'Converting symbol %s as a tag because it is more often used '
 247           'as a tag.'
 248           % (symbol,)
 249           )
 250       return Tag(symbol)
 251     else:
 252       Log().verbose(
 253           'Converting symbol %s as a branch because it is more often used '
 254           'as a branch.'
 255           % (symbol,)
 256           )
 257       return Branch(symbol)
 258
 259
 260 class _CatchAllRule(StrategyRule):
 261   """Base class for catch-all rules.
 262
 263   Usually this rule will appear after a list of more careful rules
 264   (including a general rule like UnambiguousUsageRule) and will
 265   therefore only apply to the symbols not handled earlier."""
 266
 267   def __init__(self, action):
 268     self._action = action
 269
 270   def log(self, symbol):
 271     raise NotImplementedError()
 272
 273   def get_symbol(self, symbol, stats):
 274     if isinstance(symbol, (Trunk, TypedSymbol)):
 275       return symbol
 276     else:
 277       self.log(symbol)
 278       return self._action(symbol)
 279
 280
 281 class AllBranchRule(_CatchAllRule):
 282   """Convert all symbols as branches.
 283
 284   Usually this rule will appear after a list of more careful rules
 285   (including a general rule like UnambiguousUsageRule) and will
 286   therefore only apply to the symbols not handled earlier."""
 287
 288   def __init__(self):
 289     _CatchAllRule.__init__(self, Branch)
 290
 291   def log(self, symbol):
 292     Log().verbose(
 293         'Converting symbol %s as a branch because no other rules applied.'
 294         % (symbol,)
 295         )
 296
 297
 298 class AllTagRule(_CatchAllRule):
 299   """Convert all symbols as tags.
 300
 301   We don't worry about conflicts here; they will be caught later by
 302   SymbolStatistics.check_consistency().
 303
 304   Usually this rule will appear after a list of more careful rules
 305   (including a general rule like UnambiguousUsageRule) and will
 306   therefore only apply to the symbols not handled earlier."""
 307
 308   def __init__(self):
 309     _CatchAllRule.__init__(self, Tag)
 310
 311   def log(self, symbol):
 312     Log().verbose(
 313         'Converting symbol %s as a tag because no other rules applied.'
 314         % (symbol,)
 315         )
 316
 317
 318 class TrunkPathRule(StrategyRule):
 319   """Set the base path for Trunk."""
 320
 321   def __init__(self, trunk_path):
 322     self.trunk_path = trunk_path
 323
 324   def get_symbol(self, symbol, stats):
 325     if isinstance(symbol, Trunk) and symbol.base_path is None:
 326       symbol.base_path = self.trunk_path
 327
 328     return symbol
 329
 330
 331 class SymbolPathRule(StrategyRule):
 332   """Set the base paths for symbol LODs."""
 333
 334   def __init__(self, symbol_type, base_path):
 335     self.symbol_type = symbol_type
 336     self.base_path = base_path
 337
 338   def get_symbol(self, symbol, stats):
 339     if isinstance(symbol, self.symbol_type) and symbol.base_path is None:
 340       symbol.base_path = path_join(self.base_path, symbol.name)
 341
 342     return symbol
 343
 344
 345 class BranchesPathRule(SymbolPathRule):
 346   """Set the base paths for Branch LODs."""
 347
 348   def __init__(self, branch_path):
 349     SymbolPathRule.__init__(self, Branch, branch_path)
 350
 351
 352 class TagsPathRule(SymbolPathRule):
 353   """Set the base paths for Tag LODs."""
 354
 355   def __init__(self, tag_path):
 356     SymbolPathRule.__init__(self, Tag, tag_path)
 357
 358
 359 class HeuristicPreferredParentRule(StrategyRule):
 360   """Use a heuristic rule to pick preferred parents.
 361
 362   Pick the parent that should be preferred for any TypedSymbols.  As
 363   parent, use the symbol that appeared most often as a possible parent
 364   of the symbol in question.  If multiple symbols are tied, choose the
 365   one that comes first according to the Symbol class's natural sort
 366   order."""
 367
 368   def _get_preferred_parent(self, stats):
 369     """Return the LODs that are most often possible parents in STATS.
 370
 371     Return the set of LinesOfDevelopment that appeared most often as
 372     possible parents.  The return value might contain multiple symbols
 373     if multiple LinesOfDevelopment appeared the same number of times."""
 374
 375     best_count = -1
 376     best_symbol = None
 377     for (symbol, count) in stats.possible_parents.items():
 378       if count > best_count or (count == best_count and symbol < best_symbol):
 379         best_count = count
 380         best_symbol = symbol
 381
 382     if best_symbol is None:
 383       return None
 384     else:
 385       return best_symbol
 386
 387   def get_symbol(self, symbol, stats):
 388     if isinstance(symbol, TypedSymbol) and symbol.preferred_parent_id is None:
 389       preferred_parent = self._get_preferred_parent(stats)
 390       if preferred_parent is None:
 391         Log().verbose('%s has no preferred parent' % (symbol,))
 392       else:
 393         symbol.preferred_parent_id = preferred_parent.id
 394         Log().verbose(
 395             'The preferred parent of %s is %s' % (symbol, preferred_parent,)
 396             )
 397
 398     return symbol
 399
 400
 401 class ManualTrunkRule(StrategyRule):
 402   """Change the SVN path of Trunk LODs.
 403
 404   Members:
 405
 406     project_id -- (int or None) The id of the project whose trunk
 407         should be affected by this rule.  If project_id is None, then
 408         the rule is not project-specific.
 409
 410     svn_path -- (str) The SVN path that should be used as the base
 411         directory for this trunk.  This member must not be None,
 412         though it may be the empty string for a single-project,
 413         trunk-only conversion.
 414
 415   """
 416
 417   def __init__(self, project_id, svn_path):
 418     self.project_id = project_id
 419     self.svn_path = normalize_svn_path(svn_path, allow_empty=True)
 420
 421   def get_symbol(self, symbol, stats):
 422     if (self.project_id is not None
 423         and self.project_id != stats.lod.project.id):
 424       return symbol
 425
 426     if isinstance(symbol, Trunk):
 427       symbol.base_path = self.svn_path
 428
 429     return symbol
 430
 431
 432 def convert_as_branch(symbol):
 433   Log().verbose(
 434       'Converting symbol %s as a branch because of manual setting.'
 435       % (symbol,)
 436       )
 437   return Branch(symbol)
 438
 439
 440 def convert_as_tag(symbol):
 441   Log().verbose(
 442       'Converting symbol %s as a tag because of manual setting.'
 443       % (symbol,)
 444       )
 445   return Tag(symbol)
 446
 447
 448 def exclude(symbol):
 449   Log().verbose(
 450       'Excluding symbol %s because of manual setting.'
 451       % (symbol,)
 452       )
 453   return ExcludedSymbol(symbol)
 454
 455
 456 class ManualSymbolRule(StrategyRule):
 457   """Change how particular symbols are converted.
 458
 459   Members:
 460
 461     project_id -- (int or None) The id of the project whose trunk
 462         should be affected by this rule.  If project_id is None, then
 463         the rule is not project-specific.
 464
 465     symbol_name -- (str) The name of the symbol that should be
 466         affected by this rule.
 467
 468     conversion -- (callable or None) A callable that converts the
 469         symbol to its preferred output type.  This should normally be
 470         one of (convert_as_branch, convert_as_tag, exclude).  If this
 471         member is None, then this rule does not affect the symbol's
 472         output type.
 473
 474     svn_path -- (str) The SVN path that should be used as the base
 475         directory for this trunk.  This member must not be None,
 476         though it may be the empty string for a single-project,
 477         trunk-only conversion.
 478
 479     parent_lod_name -- (str or None) The name of the line of
 480         development that should be preferred as the parent of this
 481         symbol.  (The preferred parent is the line of development from
 482         which the symbol should sprout.)  If this member is set to the
 483         string '.trunk.', then the symbol will be set to sprout
 484         directly from trunk.  If this member is set to None, then this
 485         rule won't affect the symbol's parent.
 486
 487   """
 488
 489   def __init__(
 490         self, project_id, symbol_name, conversion, svn_path, parent_lod_name
 491         ):
 492     self.project_id = project_id
 493     self.symbol_name = symbol_name
 494     self.conversion = conversion
 495     if svn_path is None:
 496       self.svn_path = None
 497     else:
 498       self.svn_path = normalize_svn_path(svn_path, allow_empty=True)
 499     self.parent_lod_name = parent_lod_name
 500
 501   def _get_parent_by_id(self, parent_lod_name, stats):
 502     """Return the LOD object for the parent with name PARENT_LOD_NAME.
 503
 504     STATS is the _Stats object describing a symbol whose parent needs
 505     to be determined from its name.  If none of its possible parents
 506     has name PARENT_LOD_NAME, raise a SymbolPlanError."""
 507
 508     for pp in stats.possible_parents.keys():
 509       if isinstance(pp, Trunk):
 510         pass
 511       elif pp.name == parent_lod_name:
 512         return pp
 513     else:
 514       parent_counts = stats.possible_parents.items()
 515       parent_counts.sort(lambda a,b: - cmp(a[1], b[1]))
 516       lines = [
 517           '%s is not a valid parent for %s;'
 518               % (parent_lod_name, stats.lod,),
 519           '    possible parents (with counts):'
 520           ]
 521       for (symbol, count) in parent_counts:
 522         if isinstance(symbol, Trunk):
 523           lines.append('        .trunk. : %d' % count)
 524         else:
 525           lines.append('        %s : %d' % (symbol.name, count))
 526       raise SymbolPlanError('\n'.join(lines))
 527
 528   def get_symbol(self, symbol, stats):
 529     if (self.project_id is not None
 530         and self.project_id != stats.lod.project.id):
 531       return symbol
 532
 533     elif isinstance(symbol, Trunk):
 534       return symbol
 535
 536     elif self.symbol_name == stats.lod.name:
 537       if self.conversion is not None:
 538         symbol = self.conversion(symbol)
 539
 540       if self.parent_lod_name is None:
 541         pass
 542       elif self.parent_lod_name == '.trunk.':
 543         symbol.preferred_parent_id = stats.lod.project.trunk_id
 544       else:
 545         symbol.preferred_parent_id = self._get_parent_by_id(
 546             self.parent_lod_name, stats
 547             ).id
 548
 549       if self.svn_path is not None:
 550         symbol.base_path = self.svn_path
 551
 552     return symbol
 553
 554
 555 class SymbolHintsFileRule(StrategyRule):
 556   """Use manual symbol configurations read from a file.
 557
 558   The input file is line-oriented with the following format:
 559
 560       <project-id> <symbol-name> <conversion> [<svn-path> [<parent-lod-name>]]
 561
 562   Where the fields are separated by whitespace and
 563
 564       project-id -- the numerical id of the Project to which the
 565           symbol belongs (numbered starting with 0).  This field can
 566           be '.' if the rule is not project-specific.
 567
 568       symbol-name -- the name of the symbol being specified, or
 569           '.trunk.' if the rule should apply to trunk.
 570
 571       conversion -- how the symbol should be treated in the
 572           conversion.  This is one of the following values: 'branch',
 573           'tag', or 'exclude'.  This field can be '.' if the rule
 574           shouldn't affect how the symbol is treated in the
 575           conversion.
 576
 577       svn-path -- the SVN path that should serve as the root path of
 578           this LOD.  The path should be expressed as a path relative
 579           to the SVN root directory, with or without a leading '/'.
 580           This field can be omitted or '.' if the rule shouldn't
 581           affect the LOD's SVN path.
 582
 583       parent-lod-name -- the name of the LOD that should serve as this
 584           symbol's parent.  This field can be omitted or '.'  if the
 585           rule shouldn't affect the symbol's parent, or it can be
 586           '.trunk.' to indicate that the symbol should sprout from the
 587           project's trunk."""
 588
 589   comment_re = re.compile(r'^(\#|$)')
 590
 591   conversion_map = {
 592       'branch' : convert_as_branch,
 593       'tag' : convert_as_tag,
 594       'exclude' : exclude,
 595       '.' : None,
 596       }
 597
 598   def __init__(self, filename):
 599     self.filename = filename
 600
 601   def start(self, symbol_statistics):
 602     self._rules = []
 603
 604     f = open(self.filename, 'r')
 605     for l in f:
 606       l = l.rstrip()
 607       s = l.lstrip()
 608       if self.comment_re.match(s):
 609         continue
 610       fields = s.split()
 611
 612       if len(fields) < 3:
 613         raise FatalError(
 614             'The following line in "%s" cannot be parsed:\n    "%s"'
 615             % (self.filename, l,)
 616             )
 617
 618       project_id = fields.pop(0)
 619       symbol_name = fields.pop(0)
 620       conversion = fields.pop(0)
 621
 622       if fields:
 623         svn_path = fields.pop(0)
 624         if svn_path == '.':
 625           svn_path = None
 626         elif svn_path[0] == '/':
 627           svn_path = svn_path[1:]
 628       else:
 629         svn_path = None
 630
 631       if fields:
 632         parent_lod_name = fields.pop(0)
 633       else:
 634         parent_lod_name = '.'
 635
 636       if fields:
 637         raise FatalError(
 638             'The following line in "%s" cannot be parsed:\n    "%s"'
 639             % (self.filename, l,)
 640             )
 641
 642       if project_id == '.':
 643         project_id = None
 644       else:
 645         try:
 646           project_id = int(project_id)
 647         except ValueError:
 648           raise FatalError(
 649               'Illegal project_id in the following line:\n    "%s"' % (l,)
 650               )
 651
 652       if symbol_name == '.trunk.':
 653         if conversion not in ['.', 'trunk']:
 654           raise FatalError('Trunk cannot be converted as a different type')
 655
 656         if parent_lod_name != '.':
 657           raise FatalError('Trunk\'s parent cannot be set')
 658
 659         if svn_path is None:
 660           # This rule doesn't do anything:
 661           pass
 662         else:
 663           self._rules.append(ManualTrunkRule(project_id, svn_path))
 664
 665       else:
 666         try:
 667           conversion = self.conversion_map[conversion]
 668         except KeyError:
 669           raise FatalError(
 670               'Illegal conversion in the following line:\n    "%s"' % (l,)
 671               )
 672
 673         if parent_lod_name == '.':
 674           parent_lod_name = None
 675
 676         if conversion is None \
 677                and svn_path is None \
 678                and parent_lod_name is None:
 679           # There is nothing to be done:
 680           pass
 681         else:
 682           self._rules.append(
 683               ManualSymbolRule(
 684                   project_id, symbol_name,
 685                   conversion, svn_path, parent_lod_name
 686                   )
 687               )
 688
 689     for rule in self._rules:
 690       rule.start(symbol_statistics)
 691
 692   def get_symbol(self, symbol, stats):
 693     for rule in self._rules:
 694       symbol = rule.get_symbol(symbol, stats)
 695
 696     return symbol
 697
 698   def finish(self):
 699     for rule in self._rules:
 700       rule.finish()
 701
 702     del self._rules
 703
 704