contrib/travis-build

   1 #!/usr/bin/env python
   2
   3 from __future__  import absolute_import, division, unicode_literals
   4
   5 import argparse
   6 import distutils.spawn
   7 import json
   8 import os
   9 import re
  10 import shlex
  11 import signal
  12 import sys
  13 import unittest
  14 import yaml
  15
  16 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  17 from cola import core
  18 from cola import git
  19 from cola import gitcmds
  20 from cola import gitcfg
  21
  22
  23 def parse_args():
  24     parser = argparse.ArgumentParser()
  25     parser.add_argument('filename', metavar='<travis.yaml>',
  26                         help='yaml travis config')
  27     parser.add_argument('--build-versions', '-V', action='append',
  28                         help='build the specified versions')
  29     parser.add_argument('--dry-run', '-k', default=False, action='store_true',
  30                         help='dry-run, do nothing')
  31     parser.add_argument('--sudo', default=False, action='store_true',
  32                         help='allow commands to use sudo')
  33     parser.add_argument('--skip-missing', default=False, action='store_true',
  34                         help='skip missing interpreters')
  35     parser.add_argument('--no-prompt', '-y', default=False, action='store_true',
  36                         help='do not prompt before each command')
  37     parser.add_argument('--default', default='y', metavar='<cmd>',
  38                         help='default command for empty input (default: y)')
  39     parser.add_argument('--remote', default='origin',
  40                         help='remote name whose URL will used for slugs')
  41     parser.add_argument('--before-install', '-b', default=False,
  42                         action='store_true',
  43                         help='run the before_install step')
  44     parser.add_argument('--before-script', '-B', default=False,
  45                         action='store_true',
  46                         help='run the before_script step')
  47     parser.add_argument('--slug', '-s', metavar='<slug>',
  48                         help='specify the URL slug')
  49     parser.add_argument('--start', metavar='<int>', type=int, default=0,
  50                         help='specify the starting command index')
  51     parser.add_argument('--test', default=False, action='store_true',
  52                         help='run built-in unit tests')
  53     parser.add_argument('--verbose', '-v', default=False, action='store_true',
  54                         help='increase verbosity')
  55     return parser.parse_args()
  56
  57
  58 def load_yaml(filename):
  59     with core.xopen(filename, 'r') as f:
  60         return yaml.load(f)
  61
  62
  63 def find_interpreter(language, version):
  64     return distutils.spawn.find_executable(language + version)
  65
  66
  67 def get_platform(platform):
  68     if platform.lower().startswith('linux'):
  69         platform = 'linux'
  70     else:
  71         platform = 'osx'
  72     return platform
  73
  74
  75 def print_error(msg):
  76     core.stderr('error: %s' % msg)
  77
  78
  79 def error(msg):
  80     print_error(msg)
  81     sys.exit(1)
  82
  83
  84 def parse_slug(url):
  85     url = url.rstrip('/')
  86     if url.endswith('.git'):
  87         url = url[:-len('.git')]
  88
  89     github_prefixes = (
  90             'https://github.com/',
  91             'git://github.com/',
  92             'git@github.com:',
  93     )
  94     for prefix in github_prefixes:
  95         if url.startswith(prefix):
  96             return url[len(prefix):]
  97
  98     patterns = (
  99             # ssh+git://user:password@host/slug
 100             r'^ssh\+git://[^@]+:[^@]+@[^@/]+/([^:]+)$',
 101             # ssh+git://user@host/slug
 102             r'^ssh\+git://[^@]+@[^@]+?/([^:]+)$',
 103             # user@host:slug (no ports, etc
 104             r'^[^@]+@[^@]+:([^:]+)$',
 105     )
 106     for pattern in patterns:
 107         rgx = re.compile(pattern)
 108         match = rgx.match(url)
 109         if match:
 110             return match.group(1)
 111
 112     return ''
 113
 114
 115 def guess_slug(remote):
 116     key = 'remote.%s.url' % remote
 117     url = gitcfg.current().get(key)
 118
 119     default_slug = 'git-cola/git-cola'
 120     if url:
 121         slug = parse_slug(url) or default_slug
 122     else:
 123         slug = default_slug
 124
 125     return slug
 126
 127
 128 def get_slug(args):
 129     if args.slug:
 130         slug = args.slug
 131     else:
 132         slug = guess_slug(args.remote)
 133     return slug
 134
 135
 136 def get_current_branch():
 137     return gitcmds.current_branch()
 138
 139
 140 def setup_environment(args, language, version):
 141     env = os.environ.copy()
 142
 143     repo = git.Git()
 144     if not repo.is_valid():
 145         error('%s is not a Git repository' % core.getcwd())
 146
 147     version_var = 'TRAVIS_%s_VERSION' % language.upper()
 148     env.setdefault(version_var, version)
 149     env.setdefault('CI', 'true')
 150     env.setdefault('TRAVIS', 'true')
 151     env.setdefault('CONTINUOUS_INTEGRATION', 'true')
 152     env.setdefault('DEBIAN_FRONTEND', 'noninteractive')
 153     env.setdefault('HAS_DAVID_A_SEAL_OF_APPROVAL', 'true')
 154     env.setdefault('USER', 'travis')
 155     env.setdefault('HOME', '/home/travis')
 156     env.setdefault('LANG', 'en_US.UTF-8')
 157     env.setdefault('LC_ALL', 'en_US.UTF-8')
 158     env.setdefault('RAILS_ENV', 'test')
 159     env.setdefault('RACK_ENV', 'test')
 160     env.setdefault('MERB_ENV', 'test')
 161     env.setdefault('TRAVIS_BRANCH', gitcmds.current_branch())
 162     env.setdefault('TRAVIS_BUILD_DIR', core.getcwd())
 163     env.setdefault('TRAVIS_BUILD_ID', '1')
 164     env.setdefault('TRAVIS_BUILD_NUMBER', '1')
 165     env.setdefault('TRAVIS_COMMIT', 'HEAD')
 166     env.setdefault('TRAVIS_COMMIT_RANGE', 'HEAD^..HEAD')
 167     env.setdefault('TRAVIS_JOB_ID', '1')
 168     env.setdefault('TRAVIS_JOB_NUMBER', '1.1')
 169     env.setdefault('TRAVIS_PULL_REQUEST', 'false')
 170     env.setdefault('TRAVIS_SECURE_ENV_VARS', 'false')
 171     env.setdefault('TRAVIS_REPO_SLUG', get_slug(args))
 172     env.setdefault('TRAVIS_OS_NAME', get_platform(sys.platform))
 173     env.setdefault('TRAVIS_TAG', '')
 174
 175     return env
 176
 177
 178 def jsonify(obj):
 179     return json.dumps(obj, sort_keys=True, indent=2)
 180
 181
 182 def print_environment(environ):
 183     print('# Environment')
 184     print(jsonify(environ))
 185
 186
 187 def expandvars(path, environ=None):
 188     """
 189     Like os.path.expandvars, but operates on a provided dictionary
 190
 191     `os.environ` is used when `environ` is None.
 192
 193     >>> expandvars('$foo', environ={'foo': 'bar'}) == 'bar'
 194     True
 195
 196     >>> environ = {'foo': 'a', 'bar': 'b'}
 197     >>> expandvars('$foo:$bar:${foo}${bar}', environ=environ)  == 'a:b:ab'
 198     True
 199
 200     """
 201     if '$' not in path:
 202         return path
 203     if environ is None:
 204         environ = os.environ.copy()
 205     try:
 206         expandvars_re = expandvars._re_cache
 207     except AttributeError:
 208         expandvars_re = expandvars._re_cache = re.compile(r'\$(\w+|\{[^}]*\})')
 209     i = 0
 210     while True:
 211         m = expandvars_re.search(path, i)
 212         if not m:
 213             break
 214         i, j = m.span(0)
 215         name = m.group(1)
 216         if name.startswith('{') and name.endswith('}'):
 217             name = name[1:-1]
 218         if name in environ:
 219             tail = path[j:]
 220             path = path[:i] + environ[name]
 221             i = len(path)
 222             path += tail
 223         else:
 224             i = j
 225     return path
 226
 227
 228 def default_shell():
 229     shell_env = os.getenv('SHELL')
 230     if shell_env:
 231         shell = shell_env
 232     else:
 233         shell = '/bin/sh'
 234     for sh in ('bash', 'zsh', 'ksh', 'sh'):
 235         executable = distutils.spawn.find_executable(sh)
 236         if executable:
 237             shell = executable
 238             break
 239     return shell
 240
 241
 242 class RunCommands(object):
 243
 244     def __init__(self, state, cmds=None):
 245         self.state = state
 246         self.cmds = cmds
 247         self.idx = state.start
 248         self.running = False
 249         self.errors = []
 250         if cmds:
 251             self.idx = min(self.idx, len(cmds)-1)
 252
 253     def loop(self, cmds=None):
 254         if cmds:
 255             self.cmds = cmds
 256         self.errors = []
 257
 258         self.running = True
 259         while self.running:
 260             self.show_prompt()
 261             self.eval_input(self.read_input())
 262
 263     def quit(self):
 264         self.running = False
 265
 266     def show_initial_prompt(self, title=None):
 267         state = self.state
 268         prompt = '# %s %s' % (state.language.title(), state.version)
 269         if title:
 270             prompt += ' - ' + title
 271         prompt += ' #'
 272         dashes = '#' * len(prompt)
 273         print(dashes)
 274         print(prompt)
 275         print(dashes)
 276         self.print_help()
 277
 278     def print_help(self):
 279         print("""
 280 *** Commands ***
 281
 282  #<command> ... repeat <command> # times, ex: "5s" skips 5
 283        y, r ... yes/run current command (default)
 284     b, k, p ... back/prev
 285  f, j, n, s ... forward/next/no/skip
 286   g, goto # ... goto command at index
 287   rw, begin ... start over from the beginning
 288     ff, end ... fast-forward to end
 289    ls, list ... list commands
 290       shell ... enter a subshell (also "bash", "zsh")
 291    cd <dir> ... change directories
 292         env ... print config-provided environment variables
 293     environ ... print current environment variables
 294         pwd ... print current working directory
 295     h, help ... show help
 296     q, quit ... quit
 297 """)
 298
 299     def prep_command(self, cmd):
 300         sudo = self.state.sudo
 301         if not sudo and cmd.startswith('sudo '):
 302             cmd = cmd[len('sudo '):]
 303         return cmd
 304
 305     def current_command(self):
 306         cmd = self.cmds[self.idx]
 307         return self.prep_command(cmd)
 308
 309     def expandvars(self, string):
 310         return expandvars(string, environ=self.state.environ)
 311
 312     def show_prompt(self):
 313         state = self.state
 314         prompt = state.prompt
 315
 316         cmd = self.current_command()
 317         expanded = self.expandvars(cmd)
 318         print('$ %s' % expanded)
 319         if prompt:
 320             sys.stdout.write('? ')
 321
 322     def read_input(self):
 323         if self.state.prompt:
 324             answer = sys.stdin.readline().strip()
 325         else:
 326             answer = 'y'
 327         return answer
 328
 329     def chdir(self, directory):
 330         os.chdir(directory)
 331         self.state.environ['PWD'] = os.getcwd()
 332
 333     def goto(self, idx):
 334         top = len(self.cmds) - 1
 335         self.idx = max(0, min(idx, top))
 336
 337     def goto_next(self):
 338         if self.is_last():
 339             self.quit()
 340             return
 341         self.goto(self.idx + 1)
 342
 343     def goto_prev(self):
 344         self.goto(self.idx - 1)
 345
 346     def rewind(self):
 347         self.goto(0)
 348
 349     def fast_forward(self):
 350         self.goto(len(self.cmds) - 1)
 351
 352     def is_last(self):
 353         return self.idx == len(self.cmds) - 1
 354
 355     def list_commands(self):
 356         for idx, cmd in enumerate(self.cmds):
 357             cmd = self.prep_command(cmd)
 358             if idx == self.idx:
 359                 decoration = '$'
 360             else:
 361                 decoration = ' '
 362             print('%03d - %s %s' % (idx, decoration, self.expandvars(cmd)))
 363
 364     def show_status(self):
 365         cmd = self.current_command()
 366         print('%03d - $ %s' % (self.idx, self.expandvars(cmd)))
 367
 368     def run_current_command(self):
 369         state = self.state
 370         dry_run = state.dry_run
 371
 372         cmd = self.current_command()
 373         argv = shlex.split(cmd)
 374         if not argv:
 375             error('empty command at index %s' % self.idx - 1)
 376
 377         if len(argv) == 2 and argv[0] == 'cd':
 378             directory = self.expandvars(argv[1])
 379             self.chdir(directory)
 380             return
 381         if dry_run:
 382             return
 383         self.run_command(cmd)
 384
 385     def run_command(self, cmd):
 386         environ = self.state.environ
 387         status, out, err = core.run_command(cmd, env=environ, shell=True,
 388                                             stdin=None, stdout=None,
 389                                             stderr=None)
 390         expanded = self.expandvars(cmd)
 391         print('The command "%s" exited with %d.' % (expanded, status))
 392         if status != 0:
 393             self.errors.append(expanded)
 394
 395     def run_shell(self, shell):
 396         self.run_command(shell)
 397
 398     def eval_input(self, answer):
 399         # Accept the action by default
 400         if not answer:
 401             answer = self.state.default
 402
 403         words = shlex.split(answer)
 404         first = words[0]
 405
 406         # Multi-command, e.g. "5s" skips 5 commands
 407         rgx = re.compile('^(\d+)(.*)$')
 408
 409         if answer in ('f', 'j', 'n', 'N', 'next', 'no', 's', 'skip'):
 410             self.goto_next()
 411             return
 412
 413         elif answer in ('b', 'k', 'p', 'back', 'prev'):
 414             self.goto_prev()
 415             return
 416
 417         elif answer in ('r', 'y', 'Y', 'yes', 'run'):
 418             self.run_current_command()
 419             self.goto_next()
 420             return
 421         elif answer in ('rw', 'rewind'):
 422             self.rewind()
 423             return
 424         elif answer in ('ff', 'last', 'end'):
 425             self.fast_forward()
 426             return
 427         elif answer in ('ls', 'list'):
 428             self.list_commands()
 429             return
 430         elif answer in ('shell',):
 431             self.run_shell(default_shell())
 432             return
 433         elif answer in ('bash', 'sh', 'zsh', 'ash', 'dash', 'ksh', 'csh'):
 434             self.run_shell(answer)
 435             return
 436         elif answer in ('st', 'status'):
 437             self.show_status()
 438             return
 439         elif answer in ('pwd',):
 440             print(os.getcwd())
 441             return
 442         elif answer in ('env',):
 443             print_environment(self.state.env)
 444             return
 445         elif answer in ('environ',):
 446             print_environment(self.state.environ)
 447             return
 448         elif first in ('g', 'go', 'goto'):
 449             self.goto(int(words[1]))
 450         elif first in ('cd', 'chdir'):
 451             self.chdir(self.expandvars(words[1]))
 452         elif answer in ('h', 'help', '?'):
 453             self.print_help()
 454             return
 455         elif answer in ('q', 'quit'):
 456             self.quit()
 457             return
 458         elif rgx.match(answer):
 459             match = rgx.match(answer)
 460             count = match.group(1)
 461             action = match.group(2)
 462             for i in range(int(count)):
 463                 self.eval_input(action)
 464             return
 465         else:
 466             print('error: %s: unknown command')
 467             self.print_help()
 468
 469
 470 class State(object):
 471
 472     def __init__(self, **kwargs):
 473         for k, v in kwargs.items():
 474             setattr(self, k, v)
 475
 476
 477
 478 def expand_config_environment(envs, environ):
 479     # Operate on a copy so that we can add to it as we expand the variables.
 480     # This allows subsequent variables to reference previous variables.
 481     expanded = []
 482     environ = environ.copy()
 483
 484     for env in envs:
 485         env_values = {}
 486         # Entries look like "VAR1=x VAR2=x$VAR1" so we split using
 487         # the shell lexer to get the individual VAR=value entries
 488         env_entries = shlex.split(env)
 489         for entry in env_entries:
 490             key, value = entry.split('=', 1)
 491             expanded_value = expandvars(value, environ=environ)
 492             # Add to the returned environment
 493             env_values[key] = expanded_value
 494             # Also add to the outer environ to allow subsequent lookups
 495             # to use a previously defined value
 496             environ[key] = expanded_value
 497         expanded.append(env_values)
 498
 499     # Ensure that we have at least a single environment
 500     if not expanded:
 501         expanded.append({})
 502
 503     return expanded
 504
 505
 506 def build_version(args, data, language, version):
 507     errors = []
 508     environ = setup_environment(args, language, version)
 509
 510     # Get the possibly config-specified execution environments
 511     envs = expand_config_environment(data.get('env', []), environ)
 512     for env in envs:
 513         environ = environ.copy()
 514         environ.update(env)
 515
 516         state = State(default=args.default,
 517                       dry_run=args.dry_run,
 518                       env=env,
 519                       environ=environ,
 520                       language=language,
 521                       prompt=not args.no_prompt,
 522                       start=args.start,
 523                       sudo=args.sudo,
 524                       verbose=args.verbose,
 525                       version=version)
 526
 527         if state.verbose:
 528             print_environment(environ)
 529
 530         cmds = []
 531         if args.before_install:
 532             cmds.extend(data.get('before_install', []))
 533         cmds.extend(data.get('install', []))
 534
 535         if args.before_script:
 536             cmds.extend(data.get('before_script', []))
 537         cmds.extend(data.get('script', []))
 538
 539         run_commands = RunCommands(state, cmds=cmds)
 540         run_commands.show_initial_prompt()
 541         run_commands.loop()
 542         errors.extend(run_commands.errors)
 543
 544     return errors
 545
 546
 547 def build(args, data):
 548     language = data['language']
 549     if args.build_versions:
 550         versions = args.build_versions
 551     else:
 552         versions = data[language]
 553
 554     if args.skip_missing:
 555         versions = [v for v in versions if find_interpreter(language, v)]
 556
 557     status = 0
 558
 559     for version in versions:
 560         errors = build_version(args, data, language, version)
 561         if errors:
 562             status = 1
 563
 564     return status
 565
 566
 567 class TravisBuildTestCase(unittest.TestCase):
 568
 569     def test_parse_slug(self):
 570         urls = (
 571             'https://github.com/example/slug',
 572             'https://github.com/example/slug.git',
 573             'git://github.com/example/slug',
 574             'git://github.com/example/slug.git',
 575             'git@github.com:example/slug',
 576             'git@github.com:example/slug.git',
 577             'git@example.com:example/slug',
 578             'git@example.com:example/slug.git',
 579             'ssh+git://git@example.com/example/slug.git',
 580             'ssh+git://user:password@example.com/example/slug.git',
 581             'ssh+git://user:password@example.com:66/example/slug.git',
 582         )
 583         expect = 'example/slug'
 584         for url in urls:
 585             actual = parse_slug(url)
 586             self.assertEqual(expect, actual)
 587
 588
 589 def test():
 590     suite = unittest.TestLoader().loadTestsFromTestCase(TravisBuildTestCase)
 591     runner = unittest.TextTestRunner(verbosity=2)
 592     result = runner.run(suite)
 593     if result.errors or result.failures:
 594         status = 1
 595     else:
 596         status = 0
 597
 598     return status
 599
 600
 601 def main():
 602     signal.signal(signal.SIGINT, signal.SIG_DFL)
 603     args = parse_args()
 604     if args.test:
 605         return test()
 606
 607     data = load_yaml(args.filename)
 608     return build(args, data)
 609
 610
 611 if __name__ == '__main__':
 612     sys.exit(main())