tools/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import argparse
   8 from datetime import datetime, timedelta
   9 import logging
  10 from operator import itemgetter
  11 import sys
  12
  13 from mach.decorators import (
  14     CommandArgument,
  15     CommandProvider,
  16     Command,
  17     SubCommand,
  18 )
  19
  20 from mozbuild.base import MachCommandBase, MozbuildObject
  21
  22
  23 def _get_busted_bugs(payload):
  24     import requests
  25     payload = dict(payload)
  26     payload['include_fields'] = 'id,summary,last_change_time,resolution'
  27     payload['blocks'] = 1543241
  28     response = requests.get('https://bugzilla.mozilla.org/rest/bug', payload)
  29     response.raise_for_status()
  30     return response.json().get('bugs', [])
  31
  32
  33 @CommandProvider
  34 class BustedProvider(MachCommandBase):
  35     @Command('busted', category='misc',
  36              description='Query known bugs in our tooling, and file new ones.')
  37     def busted_default(self):
  38         unresolved = _get_busted_bugs({'resolution': '---'})
  39         creation_time = datetime.now() - timedelta(days=15)
  40         creation_time = creation_time.strftime('%Y-%m-%dT%H-%M-%SZ')
  41         resolved = _get_busted_bugs({'creation_time': creation_time})
  42         resolved = [bug for bug in resolved if bug['resolution']]
  43         all_bugs = sorted(
  44             unresolved + resolved, key=itemgetter('last_change_time'),
  45             reverse=True)
  46         if all_bugs:
  47             for bug in all_bugs:
  48                 print("[%s] Bug %s - %s" % (
  49                     'UNRESOLVED' if not bug['resolution']
  50                     else 'RESOLVED - %s' % bug['resolution'], bug['id'],
  51                     bug['summary']))
  52         else:
  53             print("No known tooling issues found.")
  54
  55     @SubCommand('busted',
  56                 'file',
  57                 description='File a bug for busted tooling.')
  58     @CommandArgument(
  59         'against', help=(
  60             'The specific mach command that is busted (i.e. if you encountered '
  61             'an error with `mach build`, run `mach busted file build`). If '
  62             'the issue is not connected to any particular mach command, you '
  63             'can also run `mach busted file general`.'))
  64     def busted_file(self, against):
  65         import webbrowser
  66
  67         if (against != 'general' and
  68             against not in self._mach_context.commands.command_handlers):
  69             print('%s is not a valid value for `against`. `against` must be '
  70                   'the name of a `mach` command, or else the string '
  71                   '"general".' % against)
  72             return 1
  73
  74         if against == 'general':
  75             product = 'Firefox Build System'
  76             component = 'General'
  77         else:
  78             import inspect
  79             import mozpack.path as mozpath
  80
  81             # Look up the file implementing that command, then cross-refernce
  82             # moz.build files to get the product/component.
  83             handler = self._mach_context.commands.command_handlers[against]
  84             method = getattr(handler.cls, handler.method)
  85             sourcefile = mozpath.relpath(inspect.getsourcefile(method),
  86                                          self.topsrcdir)
  87             reader = self.mozbuild_reader(config_mode='empty')
  88             try:
  89                 res = reader.files_info(
  90                     [sourcefile])[sourcefile]['BUG_COMPONENT']
  91                 product, component = res.product, res.component
  92             except TypeError:
  93                 # The file might not have a bug set.
  94                 product = 'Firefox Build System'
  95                 component = 'General'
  96
  97         uri = ('https://bugzilla.mozilla.org/enter_bug.cgi?'
  98                'product=%s&component=%s&blocked=1543241' % (product, component))
  99         webbrowser.open_new_tab(uri)
 100
 101
 102 MACH_PASTEBIN_DURATIONS = {
 103     'onetime': 'onetime',
 104     'hour': '3600',
 105     'day': '86400',
 106     'week': '604800',
 107     'month': '2073600',
 108 }
 109
 110 EXTENSION_TO_HIGHLIGHTER = {
 111     '.hgrc': 'ini',
 112     'Dockerfile': 'docker',
 113     'Makefile': 'make',
 114     'applescript': 'applescript',
 115     'arduino': 'arduino',
 116     'bash': 'bash',
 117     'bat': 'bat',
 118     'c': 'c',
 119     'clojure': 'clojure',
 120     'cmake': 'cmake',
 121     'coffee': 'coffee-script',
 122     'console': 'console',
 123     'cpp': 'cpp',
 124     'cs': 'csharp',
 125     'css': 'css',
 126     'cu': 'cuda',
 127     'cuda': 'cuda',
 128     'dart': 'dart',
 129     'delphi': 'delphi',
 130     'diff': 'diff',
 131     'django': 'django',
 132     'docker': 'docker',
 133     'elixir': 'elixir',
 134     'erlang': 'erlang',
 135     'go': 'go',
 136     'h': 'c',
 137     'handlebars': 'handlebars',
 138     'haskell': 'haskell',
 139     'hs': 'haskell',
 140     'html': 'html',
 141     'ini': 'ini',
 142     'ipy': 'ipythonconsole',
 143     'ipynb': 'ipythonconsole',
 144     'irc': 'irc',
 145     'j2': 'django',
 146     'java': 'java',
 147     'js': 'js',
 148     'json': 'json',
 149     'jsx': 'jsx',
 150     'kt': 'kotlin',
 151     'less': 'less',
 152     'lisp': 'common-lisp',
 153     'lsp': 'common-lisp',
 154     'lua': 'lua',
 155     'm': 'objective-c',
 156     'make': 'make',
 157     'matlab': 'matlab',
 158     'md': '_markdown',
 159     'nginx': 'nginx',
 160     'numpy': 'numpy',
 161     'patch': 'diff',
 162     'perl': 'perl',
 163     'php': 'php',
 164     'pm': 'perl',
 165     'postgresql': 'postgresql',
 166     'py': 'python',
 167     'rb': 'rb',
 168     'rs': 'rust',
 169     'rst': 'rst',
 170     'sass': 'sass',
 171     'scss': 'scss',
 172     'sh': 'bash',
 173     'sol': 'sol',
 174     'sql': 'sql',
 175     'swift': 'swift',
 176     'tex': 'tex',
 177     'typoscript': 'typoscript',
 178     'vim': 'vim',
 179     'xml': 'xml',
 180     'xslt': 'xslt',
 181     'yaml': 'yaml',
 182     'yml': 'yaml'
 183 }
 184
 185
 186 def guess_highlighter_from_path(path):
 187     '''Return a known highlighter from a given path
 188
 189     Attempt to select a highlighter by checking the file extension in the mapping
 190     of extensions to highlighter. If that fails, attempt to pass the basename of
 191     the file. Return `_code` as the default highlighter if that fails.
 192     '''
 193     import os
 194
 195     _name, ext = os.path.splitext(path)
 196
 197     if ext.startswith('.'):
 198         ext = ext[1:]
 199
 200     if ext in EXTENSION_TO_HIGHLIGHTER:
 201         return EXTENSION_TO_HIGHLIGHTER[ext]
 202
 203     basename = os.path.basename(path)
 204
 205     return EXTENSION_TO_HIGHLIGHTER.get(basename, '_code')
 206
 207
 208 PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
 209
 210 PASTEMO_URL = 'https://paste.mozilla.org/api/'
 211
 212 MACH_PASTEBIN_DESCRIPTION = '''
 213 Command line interface to paste.mozilla.org.
 214
 215 Takes either a filename whose content should be pasted, or reads
 216 content from standard input. If a highlighter is specified it will
 217 be used, otherwise the file name will be used to determine an
 218 appropriate highlighter.
 219 '''
 220
 221
 222 @CommandProvider
 223 class PastebinProvider(MachCommandBase):
 224     @Command('pastebin', category='misc',
 225              description=MACH_PASTEBIN_DESCRIPTION)
 226     @CommandArgument('--list-highlighters', action='store_true',
 227                      help='List known highlighters and exit')
 228     @CommandArgument('--highlighter', default=None,
 229                      help='Syntax highlighting to use for paste')
 230     @CommandArgument('--expires', default='week',
 231                      choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
 232                      help='Expire paste after given time duration (default: %(default)s)')
 233     @CommandArgument('--verbose', action='store_true',
 234                      help='Print extra info such as selected syntax highlighter')
 235     @CommandArgument('path', nargs='?', default=None,
 236                      help='Path to file for upload to paste.mozilla.org')
 237     def pastebin(self, list_highlighters, highlighter, expires, verbose, path):
 238         import requests
 239
 240         def verbose_print(*args, **kwargs):
 241             '''Print a string if `--verbose` flag is set'''
 242             if verbose:
 243                 print(*args, **kwargs)
 244
 245         # Show known highlighters and exit.
 246         if list_highlighters:
 247             lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
 248             print('Available lexers:\n'
 249                   '    - %s' % '\n    - '.join(sorted(lexers)))
 250             return 0
 251
 252         # Get a correct expiry value.
 253         try:
 254             verbose_print('Setting expiry from %s' % expires)
 255             expires = MACH_PASTEBIN_DURATIONS[expires]
 256             verbose_print('Using %s as expiry' % expires)
 257         except KeyError:
 258             print('%s is not a valid duration.\n'
 259                   '(hint: try one of %s)' %
 260                   (expires, ', '.join(MACH_PASTEBIN_DURATIONS.keys())))
 261             return 1
 262
 263         data = {
 264             'format': 'json',
 265             'expires': expires,
 266         }
 267
 268         # Get content to be pasted.
 269         if path:
 270             verbose_print('Reading content from %s' % path)
 271             try:
 272                 with open(path, 'r') as f:
 273                     content = f.read()
 274             except IOError:
 275                 print('ERROR. No such file %s' % path)
 276                 return 1
 277
 278             lexer = guess_highlighter_from_path(path)
 279             if lexer:
 280                 data['lexer'] = lexer
 281         else:
 282             verbose_print('Reading content from stdin')
 283             content = sys.stdin.read()
 284
 285         # Assert the length of content to be posted does not exceed the maximum.
 286         content_length = len(content)
 287         verbose_print('Checking size of content is okay (%d)' % content_length)
 288         if content_length > PASTEMO_MAX_CONTENT_LENGTH:
 289             print('Paste content is too large (%d, maximum %d)' %
 290                   (content_length, PASTEMO_MAX_CONTENT_LENGTH))
 291             return 1
 292
 293         data['content'] = content
 294
 295         # Highlight as specified language, overwriting value set from filename.
 296         if highlighter:
 297             verbose_print('Setting %s as highlighter' % highlighter)
 298             data['lexer'] = highlighter
 299
 300         try:
 301             verbose_print('Sending request to %s' % PASTEMO_URL)
 302             resp = requests.post(PASTEMO_URL, data=data)
 303
 304             # Error code should always be 400.
 305             # Response content will include a helpful error message,
 306             # so print it here (for example, if an invalid highlighter is
 307             # provided, it will return a list of valid highlighters).
 308             if resp.status_code >= 400:
 309                 print('Error code %d: %s' % (resp.status_code, resp.content))
 310                 return 1
 311
 312             verbose_print('Pasted successfully')
 313
 314             response_json = resp.json()
 315
 316             verbose_print('Paste highlighted as %s' % response_json['lexer'])
 317             print(response_json['url'])
 318
 319             return 0
 320         except Exception as e:
 321             print('ERROR. Paste failed.')
 322             print('%s' % e)
 323         return 1
 324
 325
 326 class PypiBasedTool:
 327     """
 328     Helper for loading a tool that is hosted on pypi. The package is expected
 329     to expose a `mach_interface` module which has `new_release_on_pypi`,
 330     `parser`, and `run` functions.
 331     """
 332
 333     def __init__(self, module_name, pypi_name=None):
 334         self.name = module_name
 335         self.pypi_name = pypi_name or module_name
 336
 337     def _import(self):
 338         # Lazy loading of the tools mach interface.
 339         # Note that only the mach_interface module should be used from this file.
 340         import importlib
 341         try:
 342             return importlib.import_module('%s.mach_interface' % self.name)
 343         except ImportError:
 344             return None
 345
 346     def create_parser(self, subcommand=None):
 347         # Create the command line parser.
 348         # If the tool is not installed, or not up to date, it will
 349         # first be installed.
 350         cmd = MozbuildObject.from_environment()
 351         cmd.activate_virtualenv()
 352         tool = self._import()
 353         if not tool:
 354             # The tool is not here at all, install it
 355             cmd.virtualenv_manager.install_pip_package(self.pypi_name)
 356             print("%s was installed. please re-run your"
 357                   " command. If you keep getting this message please "
 358                   " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name))
 359         else:
 360             # Check if there is a new release available
 361             release = tool.new_release_on_pypi()
 362             if release:
 363                 print(release)
 364                 # there is one, so install it. Note that install_pip_package
 365                 # does not work here, so just run pip directly.
 366                 cmd.virtualenv_manager._run_pip([
 367                     'install',
 368                     '%s==%s' % (self.pypi_name, release)
 369                 ])
 370                 print("%s was updated to version %s. please"
 371                       " re-run your command." % (self.pypi_name, release))
 372             else:
 373                 # Tool is up to date, return the parser.
 374                 if subcommand:
 375                     return tool.parser(subcommand)
 376                 else:
 377                     return tool.parser()
 378         # exit if we updated or installed mozregression because
 379         # we may have already imported mozregression and running it
 380         # as this may cause issues.
 381         sys.exit(0)
 382
 383     def run(self, **options):
 384         tool = self._import()
 385         tool.run(options)
 386
 387
 388 def mozregression_create_parser():
 389     # Create the mozregression command line parser.
 390     # if mozregression is not installed, or not up to date, it will
 391     # first be installed.
 392     loader = PypiBasedTool("mozregression")
 393     return loader.create_parser()
 394
 395
 396 @CommandProvider
 397 class MozregressionCommand(MachCommandBase):
 398     @Command('mozregression',
 399              category='misc',
 400              description=("Regression range finder for nightly"
 401                           " and inbound builds."),
 402              parser=mozregression_create_parser)
 403     def run(self, **options):
 404         self.activate_virtualenv()
 405         mozregression = PypiBasedTool("mozregression")
 406         mozregression.run(**options)
 407
 408
 409 @CommandProvider
 410 class NodeCommands(MachCommandBase):
 411     @Command(
 412         "node",
 413         category="devenv",
 414         description="Run the NodeJS interpreter used for building.",
 415     )
 416     @CommandArgument("args", nargs=argparse.REMAINDER)
 417     def node(self, args):
 418         from mozbuild.nodeutil import find_node_executable
 419
 420         # Avoid logging the command
 421         self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 422
 423         node_path, _ = find_node_executable()
 424
 425         return self.run_process(
 426             [node_path] + args,
 427             pass_thru=True,  # Allow user to run Node interactively.
 428             ensure_exit_code=False,  # Don't throw on non-zero exit code.
 429         )
 430
 431     @Command(
 432         "npm",
 433         category="devenv",
 434         description="Run the npm executable from the NodeJS used for building.",
 435     )
 436     @CommandArgument("args", nargs=argparse.REMAINDER)
 437     def npm(self, args):
 438         from mozbuild.nodeutil import find_npm_executable
 439
 440         # Avoid logging the command
 441         self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 442
 443         npm_path, _ = find_npm_executable()
 444
 445         return self.run_process(
 446             [npm_path, "--scripts-prepend-node-path=auto"] + args,
 447             pass_thru=True,  # Avoid eating npm output/error messages
 448             ensure_exit_code=False,  # Don't throw on non-zero exit code.
 449         )
 450
 451
 452 def logspam_create_parser(subcommand):
 453     # Create the logspam command line parser.
 454     # if logspam is not installed, or not up to date, it will
 455     # first be installed.
 456     loader = PypiBasedTool("logspam", "mozilla-log-spam")
 457     return loader.create_parser(subcommand)
 458
 459
 460 from functools import partial
 461
 462
 463 @CommandProvider
 464 class LogspamCommand(MachCommandBase):
 465     @Command('logspam',
 466              category='misc',
 467              description=("Warning categorizer for treeherder test runs."))
 468     def logspam(self):
 469         pass
 470
 471     @SubCommand('logspam', 'report', parser=partial(logspam_create_parser, "report"))
 472     def report(self, **options):
 473         self.activate_virtualenv()
 474         logspam = PypiBasedTool("logspam")
 475         logspam.run(command="report", **options)
 476
 477     @SubCommand('logspam', 'bisect', parser=partial(logspam_create_parser, "bisect"))
 478     def bisect(self, **options):
 479         self.activate_virtualenv()
 480         logspam = PypiBasedTool("logspam")
 481         logspam.run(command="bisect", **options)
 482
 483     @SubCommand('logspam', 'file', parser=partial(logspam_create_parser, "file"))
 484     def create(self, **options):
 485         self.activate_virtualenv()
 486         logspam = PypiBasedTool("logspam")
 487         logspam.run(command="file", **options)