tools/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import sys
   8
   9 from mach.decorators import (
  10     CommandArgument,
  11     CommandProvider,
  12     Command,
  13     SubCommand,
  14 )
  15
  16 from mozbuild.base import MachCommandBase, MozbuildObject
  17
  18
  19 @CommandProvider
  20 class BustedProvider(MachCommandBase):
  21     @Command('busted', category='misc',
  22              description='Query known bugs in our tooling, and file new ones.')
  23     def busted_default(self):
  24         import requests
  25         payload = {'include_fields': 'id,summary,last_change_time',
  26                    'blocks': 1543241,
  27                    'resolution': '---'}
  28         response = requests.get('https://bugzilla.mozilla.org/rest/bug', payload)
  29         response.raise_for_status()
  30         json_response = response.json()
  31         if 'bugs' in json_response and len(json_response['bugs']) > 0:
  32             # Display most recently modifed bugs first.
  33             bugs = sorted(json_response['bugs'], key=lambda item: item['last_change_time'],
  34                           reverse=True)
  35             for bug in bugs:
  36                 print("Bug %s - %s" % (bug['id'], bug['summary']))
  37         else:
  38             print("No known tooling issues found.")
  39
  40     @SubCommand('busted',
  41                 'file',
  42                 description='File a bug for busted tooling.')
  43     @CommandArgument(
  44         'against', help=(
  45             'The specific mach command that is busted (i.e. if you encountered '
  46             'an error with `mach build`, run `mach busted file build`). If '
  47             'the issue is not connected to any particular mach command, you '
  48             'can also run `mach busted file general`.'))
  49     def busted_file(self, against):
  50         import webbrowser
  51
  52         if (against != 'general' and
  53             against not in self._mach_context.commands.command_handlers):
  54             print('%s is not a valid value for `against`. `against` must be '
  55                   'the name of a `mach` command, or else the string '
  56                   '"general".' % against)
  57             return 1
  58
  59         if against == 'general':
  60             product = 'Firefox Build System'
  61             component = 'General'
  62         else:
  63             import inspect
  64             import mozpack.path as mozpath
  65
  66             # Look up the file implementing that command, then cross-refernce
  67             # moz.build files to get the product/component.
  68             handler = self._mach_context.commands.command_handlers[against]
  69             method = getattr(handler.cls, handler.method)
  70             sourcefile = mozpath.relpath(inspect.getsourcefile(method),
  71                                          self.topsrcdir)
  72             reader = self.mozbuild_reader(config_mode='empty')
  73             try:
  74                 res = reader.files_info(
  75                     [sourcefile])[sourcefile]['BUG_COMPONENT']
  76                 product, component = res.product, res.component
  77             except TypeError:
  78                 # The file might not have a bug set.
  79                 product = 'Firefox Build System'
  80                 component = 'General'
  81
  82         uri = ('https://bugzilla.mozilla.org/enter_bug.cgi?'
  83                'product=%s&component=%s&blocked=1543241' % (product, component))
  84         webbrowser.open_new_tab(uri)
  85
  86
  87 @CommandProvider
  88 class UUIDProvider(object):
  89     @Command('uuid', category='misc',
  90              description='Generate a uuid.')
  91     @CommandArgument('--format', '-f', choices=['idl', 'cpp', 'c++'],
  92                      help='Output format for the generated uuid.')
  93     def uuid(self, format=None):
  94         import uuid
  95         u = uuid.uuid4()
  96         if format in [None, 'idl']:
  97             print(u)
  98             if format is None:
  99                 print('')
 100         if format in [None, 'cpp', 'c++']:
 101             u = u.hex
 102             print('{ 0x%s, 0x%s, 0x%s, \\' % (u[0:8], u[8:12], u[12:16]))
 103             pairs = tuple(map(lambda n: u[n:n+2], range(16, 32, 2)))
 104             print(('  { ' + '0x%s, ' * 7 + '0x%s } }') % pairs)
 105
 106
 107 MACH_PASTEBIN_DURATIONS = {
 108     'onetime': 'onetime',
 109     'hour': '3600',
 110     'day': '86400',
 111     'week': '604800',
 112     'month': '2073600',
 113 }
 114
 115 EXTENSION_TO_HIGHLIGHTER = {
 116     '.hgrc': 'ini',
 117     'Dockerfile': 'docker',
 118     'Makefile': 'make',
 119     'applescript': 'applescript',
 120     'arduino': 'arduino',
 121     'bash': 'bash',
 122     'bat': 'bat',
 123     'c': 'c',
 124     'clojure': 'clojure',
 125     'cmake': 'cmake',
 126     'coffee': 'coffee-script',
 127     'console': 'console',
 128     'cpp': 'cpp',
 129     'cs': 'csharp',
 130     'css': 'css',
 131     'cu': 'cuda',
 132     'cuda': 'cuda',
 133     'dart': 'dart',
 134     'delphi': 'delphi',
 135     'diff': 'diff',
 136     'django': 'django',
 137     'docker': 'docker',
 138     'elixir': 'elixir',
 139     'erlang': 'erlang',
 140     'go': 'go',
 141     'h': 'c',
 142     'handlebars': 'handlebars',
 143     'haskell': 'haskell',
 144     'hs': 'haskell',
 145     'html': 'html',
 146     'ini': 'ini',
 147     'ipy': 'ipythonconsole',
 148     'ipynb': 'ipythonconsole',
 149     'irc': 'irc',
 150     'j2': 'django',
 151     'java': 'java',
 152     'js': 'js',
 153     'json': 'json',
 154     'jsx': 'jsx',
 155     'kt': 'kotlin',
 156     'less': 'less',
 157     'lisp': 'common-lisp',
 158     'lsp': 'common-lisp',
 159     'lua': 'lua',
 160     'm': 'objective-c',
 161     'make': 'make',
 162     'matlab': 'matlab',
 163     'md': '_markdown',
 164     'nginx': 'nginx',
 165     'numpy': 'numpy',
 166     'patch': 'diff',
 167     'perl': 'perl',
 168     'php': 'php',
 169     'pm': 'perl',
 170     'postgresql': 'postgresql',
 171     'py': 'python',
 172     'rb': 'rb',
 173     'rs': 'rust',
 174     'rst': 'rst',
 175     'sass': 'sass',
 176     'scss': 'scss',
 177     'sh': 'bash',
 178     'sol': 'sol',
 179     'sql': 'sql',
 180     'swift': 'swift',
 181     'tex': 'tex',
 182     'typoscript': 'typoscript',
 183     'vim': 'vim',
 184     'xml': 'xml',
 185     'xslt': 'xslt',
 186     'yaml': 'yaml',
 187     'yml': 'yaml'
 188 }
 189
 190
 191 def guess_highlighter_from_path(path):
 192     '''Return a known highlighter from a given path
 193
 194     Attempt to select a highlighter by checking the file extension in the mapping
 195     of extensions to highlighter. If that fails, attempt to pass the basename of
 196     the file. Return `_code` as the default highlighter if that fails.
 197     '''
 198     import os
 199
 200     _name, ext = os.path.splitext(path)
 201
 202     if ext.startswith('.'):
 203         ext = ext[1:]
 204
 205     if ext in EXTENSION_TO_HIGHLIGHTER:
 206         return EXTENSION_TO_HIGHLIGHTER[ext]
 207
 208     basename = os.path.basename(path)
 209
 210     return EXTENSION_TO_HIGHLIGHTER.get(basename, '_code')
 211
 212
 213 PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
 214
 215 PASTEMO_URL = 'https://paste.mozilla.org/api/'
 216
 217 MACH_PASTEBIN_DESCRIPTION = '''
 218 Command line interface to paste.mozilla.org.
 219
 220 Takes either a filename whose content should be pasted, or reads
 221 content from standard input. If a highlighter is specified it will
 222 be used, otherwise the file name will be used to determine an
 223 appropriate highlighter.
 224 '''
 225
 226
 227 @CommandProvider
 228 class PastebinProvider(object):
 229     @Command('pastebin', category='misc',
 230              description=MACH_PASTEBIN_DESCRIPTION)
 231     @CommandArgument('--list-highlighters', action='store_true',
 232                      help='List known highlighters and exit')
 233     @CommandArgument('--highlighter', default=None,
 234                      help='Syntax highlighting to use for paste')
 235     @CommandArgument('--expires', default='week',
 236                      choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
 237                      help='Expire paste after given time duration (default: %(default)s)')
 238     @CommandArgument('--verbose', action='store_true',
 239                      help='Print extra info such as selected syntax highlighter')
 240     @CommandArgument('path', nargs='?', default=None,
 241                      help='Path to file for upload to paste.mozilla.org')
 242     def pastebin(self, list_highlighters, highlighter, expires, verbose, path):
 243         import requests
 244
 245         def verbose_print(*args, **kwargs):
 246             '''Print a string if `--verbose` flag is set'''
 247             if verbose:
 248                 print(*args, **kwargs)
 249
 250         # Show known highlighters and exit.
 251         if list_highlighters:
 252             lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
 253             print('Available lexers:\n'
 254                   '    - %s' % '\n    - '.join(sorted(lexers)))
 255             return 0
 256
 257         # Get a correct expiry value.
 258         try:
 259             verbose_print('Setting expiry from %s' % expires)
 260             expires = MACH_PASTEBIN_DURATIONS[expires]
 261             verbose_print('Using %s as expiry' % expires)
 262         except KeyError:
 263             print('%s is not a valid duration.\n'
 264                   '(hint: try one of %s)' %
 265                   (expires, ', '.join(MACH_PASTEBIN_DURATIONS.keys())))
 266             return 1
 267
 268         data = {
 269             'format': 'json',
 270             'expires': expires,
 271         }
 272
 273         # Get content to be pasted.
 274         if path:
 275             verbose_print('Reading content from %s' % path)
 276             try:
 277                 with open(path, 'r') as f:
 278                     content = f.read()
 279             except IOError:
 280                 print('ERROR. No such file %s' % path)
 281                 return 1
 282
 283             lexer = guess_highlighter_from_path(path)
 284             if lexer:
 285                 data['lexer'] = lexer
 286         else:
 287             verbose_print('Reading content from stdin')
 288             content = sys.stdin.read()
 289
 290         # Assert the length of content to be posted does not exceed the maximum.
 291         content_length = len(content)
 292         verbose_print('Checking size of content is okay (%d)' % content_length)
 293         if content_length > PASTEMO_MAX_CONTENT_LENGTH:
 294             print('Paste content is too large (%d, maximum %d)' %
 295                   (content_length, PASTEMO_MAX_CONTENT_LENGTH))
 296             return 1
 297
 298         data['content'] = content
 299
 300         # Highlight as specified language, overwriting value set from filename.
 301         if highlighter:
 302             verbose_print('Setting %s as highlighter' % highlighter)
 303             data['lexer'] = highlighter
 304
 305         try:
 306             verbose_print('Sending request to %s' % PASTEMO_URL)
 307             resp = requests.post(PASTEMO_URL, data=data)
 308
 309             # Error code should always be 400.
 310             # Response content will include a helpful error message,
 311             # so print it here (for example, if an invalid highlighter is
 312             # provided, it will return a list of valid highlighters).
 313             if resp.status_code >= 400:
 314                 print('Error code %d: %s' % (resp.status_code, resp.content))
 315                 return 1
 316
 317             verbose_print('Pasted successfully')
 318
 319             response_json = resp.json()
 320
 321             verbose_print('Paste highlighted as %s' % response_json['lexer'])
 322             print(response_json['url'])
 323
 324             return 0
 325         except Exception as e:
 326             print('ERROR. Paste failed.')
 327             print('%s' % e)
 328         return 1
 329
 330
 331 def mozregression_import():
 332     # Lazy loading of mozregression.
 333     # Note that only the mach_interface module should be used from this file.
 334     try:
 335         import mozregression.mach_interface
 336     except ImportError:
 337         return None
 338     return mozregression.mach_interface
 339
 340
 341 def mozregression_create_parser():
 342     # Create the mozregression command line parser.
 343     # if mozregression is not installed, or not up to date, it will
 344     # first be installed.
 345     cmd = MozbuildObject.from_environment()
 346     cmd._activate_virtualenv()
 347     mozregression = mozregression_import()
 348     if not mozregression:
 349         # mozregression is not here at all, install it
 350         cmd.virtualenv_manager.install_pip_package('mozregression')
 351         print("mozregression was installed. please re-run your"
 352               " command. If you keep getting this message please "
 353               " manually run: 'pip install -U mozregression'.")
 354     else:
 355         # check if there is a new release available
 356         release = mozregression.new_release_on_pypi()
 357         if release:
 358             print(release)
 359             # there is one, so install it. Note that install_pip_package
 360             # does not work here, so just run pip directly.
 361             cmd.virtualenv_manager._run_pip([
 362                 'install',
 363                 'mozregression==%s' % release
 364             ])
 365             print("mozregression was updated to version %s. please"
 366                   " re-run your command." % release)
 367         else:
 368             # mozregression is up to date, return the parser.
 369             return mozregression.parser()
 370     # exit if we updated or installed mozregression because
 371     # we may have already imported mozregression and running it
 372     # as this may cause issues.
 373     sys.exit(0)
 374
 375
 376 @CommandProvider
 377 class MozregressionCommand(MachCommandBase):
 378     @Command('mozregression',
 379              category='misc',
 380              description=("Regression range finder for nightly"
 381                           " and inbound builds."),
 382              parser=mozregression_create_parser)
 383     def run(self, **options):
 384         self._activate_virtualenv()
 385         mozregression = mozregression_import()
 386         mozregression.run(options)