contrib/travis-build

   1 #!/usr/bin/env python
   2 # flake8: noqa
   3 from __future__ import absolute_import, division, unicode_literals
   4 from __future__ import print_function
   5 import argparse
   6 import json
   7 import os
   8 import re
   9 import shlex
  10 import signal
  11 import sys
  12 import unittest
  13 import yaml
  14
  15 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  16 from cola import core
  17 from cola import git
  18 from cola import gitcmds
  19 from cola import gitcfg
  20
  21
  22 def parse_args():
  23     parser = argparse.ArgumentParser()
  24     parser.add_argument('filename', metavar='<travis.yaml>',
  25                         help='yaml travis config')
  26     parser.add_argument('--build-versions', '-V', action='append',
  27                         help='build the specified versions')
  28     parser.add_argument('--dry-run', '-k', default=False, action='store_true',
  29                         help='dry-run, do nothing')
  30     parser.add_argument('--sudo', default=False, action='store_true',
  31                         help='allow commands to use sudo')
  32     parser.add_argument('--skip-missing', default=False, action='store_true',
  33                         help='skip missing interpreters')
  34     parser.add_argument('--no-prompt', '-y', default=False, action='store_true',
  35                         help='do not prompt before each command')
  36     parser.add_argument('--default', default='y', metavar='<cmd>',
  37                         help='default command for empty input (default: y)')
  38     parser.add_argument('--remote', default='origin',
  39                         help='remote name whose URL will used for slugs')
  40     parser.add_argument('--before-install', '-b', default=False,
  41                         action='store_true',
  42                         help='run the before_install step')
  43     parser.add_argument('--before-script', '-B', default=False,
  44                         action='store_true',
  45                         help='run the before_script step')
  46     parser.add_argument('--slug', '-s', metavar='<slug>',
  47                         help='specify the URL slug')
  48     parser.add_argument('--start', metavar='<int>', type=int, default=0,
  49                         help='specify the starting command index')
  50     parser.add_argument('--test', default=False, action='store_true',
  51                         help='run built-in unit tests')
  52     parser.add_argument('--verbose', '-v', default=False, action='store_true',
  53                         help='increase verbosity')
  54     return parser.parse_args()
  55
  56
  57 def load_yaml(filename):
  58     with core.xopen(filename, 'r') as f:
  59         return yaml.load(f)
  60
  61
  62 def find_interpreter(language, version):
  63     return core.find_executable(language + version)
  64
  65
  66 def get_platform(platform):
  67     if platform.lower().startswith('linux'):
  68         platform = 'linux'
  69     else:
  70         platform = 'osx'
  71     return platform
  72
  73
  74 def print_error(msg):
  75     core.print_stderr('error: %s' % msg)
  76
  77
  78 def error(msg):
  79     print_error(msg)
  80     sys.exit(1)
  81
  82
  83 def parse_slug(url):
  84     url = url.rstrip('/')
  85     if url.endswith('.git'):
  86         url = url[:-len('.git')]
  87
  88     github_prefixes = (
  89         'https://github.com/',
  90         'git://github.com/',
  91         'git@github.com:',
  92     )
  93     for prefix in github_prefixes:
  94         if url.startswith(prefix):
  95             return url[len(prefix):]
  96
  97     patterns = (
  98         # ssh+git://user:password@host/slug
  99         r'^ssh\+git://[^@]+:[^@]+@[^@/]+/([^:]+)$',
 100         # ssh+git://user@host/slug
 101         r'^ssh\+git://[^@]+@[^@]+?/([^:]+)$',
 102         # user@host:slug (no ports, etc
 103         r'^[^@]+@[^@]+:([^:]+)$',
 104     )
 105     for pattern in patterns:
 106         rgx = re.compile(pattern)
 107         match = rgx.match(url)
 108         if match:
 109             return match.group(1)
 110
 111     return ''
 112
 113
 114 def guess_slug(context, remote):
 115     key = 'remote.%s.url' % remote
 116     url = context.cfg.get(key)
 117
 118     default_slug = 'git-cola/git-cola'
 119     if url:
 120         slug = parse_slug(url) or default_slug
 121     else:
 122         slug = default_slug
 123
 124     return slug
 125
 126
 127 def get_slug(context, args):
 128     if args.slug:
 129         slug = args.slug
 130     else:
 131         slug = guess_slug(context, args.remote)
 132     return slug
 133
 134
 135 class ApplicationContext(object):
 136
 137     def __init__(self, args):
 138         self.args = args
 139         self.git = None
 140         self.cfg = None
 141
 142
 143 def new_context(args):
 144     """Create top-level ApplicationContext objects"""
 145     context = ApplicationContext(args)
 146     context.git = git.create()
 147     context.cfg = gitcfg.create(context)
 148     return context
 149
 150
 151 def setup_environment(args, language, version):
 152     env = os.environ.copy()
 153
 154     repo = git.Git()
 155     if not repo.is_valid():
 156         error('%s is not a Git repository' % core.getcwd())
 157
 158     context = new_context(args)
 159
 160     version_var = 'TRAVIS_%s_VERSION' % language.upper()
 161     env.setdefault(version_var, version)
 162     env.setdefault('CI', 'true')
 163     env.setdefault('TRAVIS', 'true')
 164     env.setdefault('CONTINUOUS_INTEGRATION', 'true')
 165     env.setdefault('DEBIAN_FRONTEND', 'noninteractive')
 166     env.setdefault('HAS_DAVID_A_SEAL_OF_APPROVAL', 'true')
 167     env.setdefault('USER', 'travis')
 168     env.setdefault('HOME', '/home/travis')
 169     env.setdefault('LANG', 'en_US.UTF-8')
 170     env.setdefault('LC_ALL', 'en_US.UTF-8')
 171     env.setdefault('RAILS_ENV', 'test')
 172     env.setdefault('RACK_ENV', 'test')
 173     env.setdefault('MERB_ENV', 'test')
 174     env.setdefault('TRAVIS_BRANCH', gitcmds.current_branch(context))
 175     env.setdefault('TRAVIS_BUILD_DIR', core.getcwd())
 176     env.setdefault('TRAVIS_BUILD_ID', '1')
 177     env.setdefault('TRAVIS_BUILD_NUMBER', '1')
 178     env.setdefault('TRAVIS_COMMIT', 'HEAD')
 179     env.setdefault('TRAVIS_COMMIT_RANGE', 'HEAD^..HEAD')
 180     env.setdefault('TRAVIS_JOB_ID', '1')
 181     env.setdefault('TRAVIS_JOB_NUMBER', '1.1')
 182     env.setdefault('TRAVIS_PULL_REQUEST', 'false')
 183     env.setdefault('TRAVIS_SECURE_ENV_VARS', 'false')
 184     env.setdefault('TRAVIS_REPO_SLUG', get_slug(context, args))
 185     env.setdefault('TRAVIS_OS_NAME', get_platform(sys.platform))
 186     env.setdefault('TRAVIS_TAG', '')
 187
 188     return env
 189
 190
 191 def jsonify(obj):
 192     return json.dumps(obj, sort_keys=True, indent=2)
 193
 194
 195 def print_environment(environ):
 196     print('# Environment')
 197     print(jsonify(environ))
 198
 199
 200 def expandvars(path, environ=None):
 201     """
 202     Like os.path.expandvars, but operates on a provided dictionary
 203
 204     `os.environ` is used when `environ` is None.
 205
 206     >>> expandvars('$foo', environ={'foo': 'bar'}) == 'bar'
 207     True
 208
 209     >>> environ = {'foo': 'a', 'bar': 'b'}
 210     >>> expandvars('$foo:$bar:${foo}${bar}', environ=environ)  == 'a:b:ab'
 211     True
 212
 213     """
 214     if '$' not in path:
 215         return path
 216     if environ is None:
 217         environ = os.environ.copy()
 218     # pylint: disable=protected-access
 219     try:
 220         expandvars_re = expandvars._re_cache
 221     except AttributeError:
 222         expandvars_re = expandvars._re_cache = re.compile(r'\$(\w+|\{[^}]*\})')
 223     i = 0
 224     while True:
 225         m = expandvars_re.search(path, i)
 226         if not m:
 227             break
 228         i, j = m.span(0)
 229         name = m.group(1)
 230         if name.startswith('{') and name.endswith('}'):
 231             name = name[1:-1]
 232         if name in environ:
 233             tail = path[j:]
 234             path = path[:i] + environ[name]
 235             i = len(path)
 236             path += tail
 237         else:
 238             i = j
 239     return path
 240
 241
 242 def default_shell():
 243     shell_env = os.getenv('SHELL')
 244     if shell_env:
 245         shell = shell_env
 246     else:
 247         shell = '/bin/sh'
 248     for sh in ('bash', 'zsh', 'ksh', 'sh'):
 249         executable = core.find_executable(sh)
 250         if executable:
 251             shell = executable
 252             break
 253     return shell
 254
 255
 256 class RunCommands(object):
 257
 258     def __init__(self, state, cmds=None):
 259         self.state = state
 260         self.cmds = cmds
 261         self.idx = state.start
 262         self.running = False
 263         self.errors = []
 264         if cmds:
 265             self.idx = min(self.idx, len(cmds)-1)
 266
 267     def loop(self, cmds=None):
 268         if cmds:
 269             self.cmds = cmds
 270         self.errors = []
 271
 272         self.running = True
 273         while self.running:
 274             self.show_prompt()
 275             self.eval_input(self.read_input())
 276
 277     def quit(self):
 278         self.running = False
 279
 280     def show_initial_prompt(self, title=None):
 281         state = self.state
 282         prompt = '# %s %s' % (state.language.title(), state.version)
 283         if title:
 284             prompt += ' - ' + title
 285         prompt += ' #'
 286         dashes = '#' * len(prompt)
 287         print(dashes)
 288         print(prompt)
 289         print(dashes)
 290         self.print_help()
 291
 292     def print_help(self):
 293         print("""
 294 *** Commands ***
 295
 296  #<command> ... repeat <command> # times, ex: "5s" skips 5
 297        y, r ... yes/run current command (default)
 298     b, k, p ... back/prev
 299  f, j, n, s ... forward/next/no/skip
 300   g, goto # ... goto command at index
 301   rw, begin ... start over from the beginning
 302     ff, end ... fast-forward to end
 303    ls, list ... list commands
 304       shell ... enter a subshell (also "bash", "zsh")
 305    cd <dir> ... change directories
 306         env ... print config-provided environment variables
 307     environ ... print current environment variables
 308         pwd ... print current working directory
 309     h, help ... show help
 310     q, quit ... quit
 311 """)
 312
 313     def prep_command(self, cmd):
 314         sudo = self.state.sudo
 315         if not sudo and cmd.startswith('sudo '):
 316             cmd = cmd[len('sudo '):]
 317         return cmd
 318
 319     def current_command(self):
 320         cmd = self.cmds[self.idx]
 321         return self.prep_command(cmd)
 322
 323     def expandvars(self, string):
 324         return expandvars(string, environ=self.state.environ)
 325
 326     def show_prompt(self):
 327         state = self.state
 328         prompt = state.prompt
 329
 330         cmd = self.current_command()
 331         expanded = self.expandvars(cmd)
 332         print('$ %s' % expanded)
 333         if prompt:
 334             sys.stdout.write('? ')
 335
 336     def read_input(self):
 337         if self.state.prompt:
 338             answer = sys.stdin.readline().strip()
 339         else:
 340             answer = 'y'
 341         return answer
 342
 343     def chdir(self, directory):
 344         os.chdir(directory)
 345         self.state.environ['PWD'] = os.getcwd()
 346
 347     def goto(self, idx):
 348         top = len(self.cmds) - 1
 349         self.idx = max(0, min(idx, top))
 350
 351     def goto_next(self):
 352         if self.is_last():
 353             self.quit()
 354             return
 355         self.goto(self.idx + 1)
 356
 357     def goto_prev(self):
 358         self.goto(self.idx - 1)
 359
 360     def rewind(self):
 361         self.goto(0)
 362
 363     def fast_forward(self):
 364         self.goto(len(self.cmds) - 1)
 365
 366     def is_last(self):
 367         return self.idx == len(self.cmds) - 1
 368
 369     def list_commands(self):
 370         for idx, cmd in enumerate(self.cmds):
 371             cmd = self.prep_command(cmd)
 372             if idx == self.idx:
 373                 decoration = '$'
 374             else:
 375                 decoration = ' '
 376             print('%03d - %s %s' % (idx, decoration, self.expandvars(cmd)))
 377
 378     def show_status(self):
 379         cmd = self.current_command()
 380         print('%03d - $ %s' % (self.idx, self.expandvars(cmd)))
 381
 382     def run_current_command(self):
 383         state = self.state
 384         dry_run = state.dry_run
 385
 386         cmd = self.current_command()
 387         argv = shlex.split(cmd)
 388         if not argv:
 389             error('empty command at index %s' % self.idx - 1)
 390
 391         if len(argv) == 2 and argv[0] == 'cd':
 392             directory = self.expandvars(argv[1])
 393             self.chdir(directory)
 394             return
 395         if dry_run:
 396             return
 397         self.run_command(cmd)
 398
 399     def run_command(self, cmd):
 400         environ = self.state.environ
 401         status, _, _ = core.run_command(cmd, env=environ, shell=True,
 402                                         stdin=None, stdout=None, stderr=None)
 403         expanded = self.expandvars(cmd)
 404         print('The command "%s" exited with %d.' % (expanded, status))
 405         if status != 0:
 406             self.errors.append(expanded)
 407
 408     def run_shell(self, shell):
 409         self.run_command(shell)
 410
 411     def eval_input(self, answer):
 412         # Accept the action by default
 413         if not answer:
 414             answer = self.state.default
 415
 416         words = shlex.split(answer)
 417         first = words[0]
 418
 419         # Multi-command, e.g. "5s" skips 5 commands
 420         rgx = re.compile(r'^(\d+)(.*)$')
 421
 422         if answer in ('f', 'j', 'n', 'N', 'next', 'no', 's', 'skip'):
 423             self.goto_next()
 424
 425         elif answer in ('b', 'k', 'p', 'back', 'prev'):
 426             self.goto_prev()
 427
 428         elif answer in ('r', 'y', 'Y', 'yes', 'run'):
 429             self.run_current_command()
 430             self.goto_next()
 431
 432         elif answer in ('rw', 'rewind'):
 433             self.rewind()
 434
 435         elif answer in ('ff', 'last', 'end'):
 436             self.fast_forward()
 437
 438         elif answer in ('ls', 'list'):
 439             self.list_commands()
 440
 441         elif answer in ('shell',):
 442             self.run_shell(default_shell())
 443
 444         elif answer in ('bash', 'sh', 'zsh', 'ash', 'dash', 'ksh', 'csh'):
 445             self.run_shell(answer)
 446
 447         elif answer in ('st', 'status'):
 448             self.show_status()
 449
 450         elif answer in ('pwd',):
 451             print(os.getcwd())
 452
 453         elif answer in ('env',):
 454             print_environment(self.state.env)
 455
 456         elif answer in ('environ',):
 457             print_environment(self.state.environ)
 458
 459         elif first in ('g', 'go', 'goto'):
 460             self.goto(int(words[1]))
 461
 462         elif first in ('cd', 'chdir'):
 463             self.chdir(self.expandvars(words[1]))
 464
 465         elif answer in ('h', 'help', '?'):
 466             self.print_help()
 467
 468         elif answer in ('q', 'quit'):
 469             self.quit()
 470
 471         elif rgx.match(answer):
 472             match = rgx.match(answer)
 473             count = match.group(1)
 474             action = match.group(2)
 475             for _ in range(int(count)):
 476                 self.eval_input(action)
 477         else:
 478             print_error('%s: unknown command' % answer)
 479             self.print_help()
 480
 481
 482 class State(object):
 483
 484     def __init__(self, **kwargs):
 485         for k, v in kwargs.items():
 486             setattr(self, k, v)
 487
 488
 489
 490 def expand_config_environment(envs, environ):
 491     # Operate on a copy so that we can add to it as we expand the variables.
 492     # This allows subsequent variables to reference previous variables.
 493     expanded = []
 494     environ = environ.copy()
 495
 496     for env in envs:
 497         env_values = {}
 498         # Entries look like "VAR1=x VAR2=x$VAR1" so we split using
 499         # the shell lexer to get the individual VAR=value entries
 500         env_entries = shlex.split(env)
 501         for entry in env_entries:
 502             key, value = entry.split('=', 1)
 503             expanded_value = expandvars(value, environ=environ)
 504             # Add to the returned environment
 505             env_values[key] = expanded_value
 506             # Also add to the outer environ to allow subsequent lookups
 507             # to use a previously defined value
 508             environ[key] = expanded_value
 509         expanded.append(env_values)
 510
 511     # Ensure that we have at least a single environment
 512     if not expanded:
 513         expanded.append({})
 514
 515     return expanded
 516
 517
 518 def build_version(args, data, language, version):
 519     errors = []
 520     environ = setup_environment(args, language, version)
 521
 522     # Get the possibly config-specified execution environments
 523     envs = expand_config_environment(data.get('env', []), environ)
 524     for env in envs:
 525         environ = environ.copy()
 526         environ.update(env)
 527
 528         state = State(default=args.default,
 529                       dry_run=args.dry_run,
 530                       env=env,
 531                       environ=environ,
 532                       language=language,
 533                       prompt=not args.no_prompt,
 534                       start=args.start,
 535                       sudo=args.sudo,
 536                       verbose=args.verbose,
 537                       version=version)
 538
 539         # pylint: disable=no-member
 540         if state.verbose:
 541             print_environment(environ)
 542
 543         cmds = []
 544         if args.before_install:
 545             cmds.extend(data.get('before_install', []))
 546         cmds.extend(data.get('install', []))
 547
 548         if args.before_script:
 549             cmds.extend(data.get('before_script', []))
 550         cmds.extend(data.get('script', []))
 551
 552         run_commands = RunCommands(state, cmds=cmds)
 553         run_commands.show_initial_prompt()
 554         run_commands.loop()
 555         errors.extend(run_commands.errors)
 556
 557     return errors
 558
 559
 560 def build(args, data):
 561     language = data['language']
 562     if args.build_versions:
 563         versions = args.build_versions
 564     else:
 565         versions = data[language]
 566
 567     if args.skip_missing:
 568         versions = [v for v in versions if find_interpreter(language, v)]
 569
 570     status = 0
 571
 572     for version in versions:
 573         errors = build_version(args, data, language, version)
 574         if errors:
 575             status = 1
 576
 577     return status
 578
 579
 580 class TravisBuildTestCase(unittest.TestCase):
 581
 582     def test_parse_slug(self):
 583         urls = (
 584             'https://github.com/example/slug',
 585             'https://github.com/example/slug.git',
 586             'git://github.com/example/slug',
 587             'git://github.com/example/slug.git',
 588             'git@github.com:example/slug',
 589             'git@github.com:example/slug.git',
 590             'git@example.com:example/slug',
 591             'git@example.com:example/slug.git',
 592             'ssh+git://git@example.com/example/slug.git',
 593             'ssh+git://user:password@example.com/example/slug.git',
 594             'ssh+git://user:password@example.com:66/example/slug.git',
 595         )
 596         expect = 'example/slug'
 597         for url in urls:
 598             actual = parse_slug(url)
 599             self.assertEqual(expect, actual)
 600
 601
 602 def test():
 603     suite = unittest.TestLoader().loadTestsFromTestCase(TravisBuildTestCase)
 604     runner = unittest.TextTestRunner(verbosity=2)
 605     result = runner.run(suite)
 606     if result.errors or result.failures:
 607         status = 1
 608     else:
 609         status = 0
 610
 611     return status
 612
 613
 614 def main():
 615     signal.signal(signal.SIGINT, signal.SIG_DFL)
 616     args = parse_args()
 617     if args.test:
 618         return test()
 619
 620     data = load_yaml(args.filename)
 621     return build(args, data)
 622
 623
 624 if __name__ == '__main__':
 625     sys.exit(main())