tools/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import argparse
   8 from datetime import datetime, timedelta
   9 import logging
  10 from operator import itemgetter
  11 import sys
  12
  13 from mach.decorators import (
  14     CommandArgument,
  15     CommandProvider,
  16     Command,
  17     SubCommand,
  18 )
  19
  20 from mozbuild.base import MachCommandBase, MozbuildObject
  21
  22
  23 def _get_busted_bugs(payload):
  24     import requests
  25
  26     payload = dict(payload)
  27     payload["include_fields"] = "id,summary,last_change_time,resolution"
  28     payload["blocks"] = 1543241
  29     response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload)
  30     response.raise_for_status()
  31     return response.json().get("bugs", [])
  32
  33
  34 @CommandProvider
  35 class BustedProvider(MachCommandBase):
  36     @Command(
  37         "busted",
  38         category="misc",
  39         description="Query known bugs in our tooling, and file new ones.",
  40     )
  41     def busted_default(self, command_context):
  42         unresolved = _get_busted_bugs({"resolution": "---"})
  43         creation_time = datetime.now() - timedelta(days=15)
  44         creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ")
  45         resolved = _get_busted_bugs({"creation_time": creation_time})
  46         resolved = [bug for bug in resolved if bug["resolution"]]
  47         all_bugs = sorted(
  48             unresolved + resolved, key=itemgetter("last_change_time"), reverse=True
  49         )
  50         if all_bugs:
  51             for bug in all_bugs:
  52                 print(
  53                     "[%s] Bug %s - %s"
  54                     % (
  55                         "UNRESOLVED"
  56                         if not bug["resolution"]
  57                         else "RESOLVED - %s" % bug["resolution"],
  58                         bug["id"],
  59                         bug["summary"],
  60                     )
  61                 )
  62         else:
  63             print("No known tooling issues found.")
  64
  65     @SubCommand("busted", "file", description="File a bug for busted tooling.")
  66     @CommandArgument(
  67         "against",
  68         help=(
  69             "The specific mach command that is busted (i.e. if you encountered "
  70             "an error with `mach build`, run `mach busted file build`). If "
  71             "the issue is not connected to any particular mach command, you "
  72             "can also run `mach busted file general`."
  73         ),
  74     )
  75     def busted_file(self, command_context, against):
  76         import webbrowser
  77
  78         if (
  79             against != "general"
  80             and against not in command_context._mach_context.commands.command_handlers
  81         ):
  82             print(
  83                 "%s is not a valid value for `against`. `against` must be "
  84                 "the name of a `mach` command, or else the string "
  85                 '"general".' % against
  86             )
  87             return 1
  88
  89         if against == "general":
  90             product = "Firefox Build System"
  91             component = "General"
  92         else:
  93             import inspect
  94             import mozpack.path as mozpath
  95
  96             # Look up the file implementing that command, then cross-refernce
  97             # moz.build files to get the product/component.
  98             handler = command_context._mach_context.commands.command_handlers[against]
  99             method = getattr(handler.cls, handler.method)
 100             sourcefile = mozpath.relpath(
 101                 inspect.getsourcefile(method), command_context.topsrcdir
 102             )
 103             reader = command_context.mozbuild_reader(config_mode="empty")
 104             try:
 105                 res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
 106                 product, component = res.product, res.component
 107             except TypeError:
 108                 # The file might not have a bug set.
 109                 product = "Firefox Build System"
 110                 component = "General"
 111
 112         uri = (
 113             "https://bugzilla.mozilla.org/enter_bug.cgi?"
 114             "product=%s&component=%s&blocked=1543241" % (product, component)
 115         )
 116         webbrowser.open_new_tab(uri)
 117
 118
 119 MACH_PASTEBIN_DURATIONS = {
 120     "onetime": "onetime",
 121     "hour": "3600",
 122     "day": "86400",
 123     "week": "604800",
 124     "month": "2073600",
 125 }
 126
 127 EXTENSION_TO_HIGHLIGHTER = {
 128     ".hgrc": "ini",
 129     "Dockerfile": "docker",
 130     "Makefile": "make",
 131     "applescript": "applescript",
 132     "arduino": "arduino",
 133     "bash": "bash",
 134     "bat": "bat",
 135     "c": "c",
 136     "clojure": "clojure",
 137     "cmake": "cmake",
 138     "coffee": "coffee-script",
 139     "console": "console",
 140     "cpp": "cpp",
 141     "cs": "csharp",
 142     "css": "css",
 143     "cu": "cuda",
 144     "cuda": "cuda",
 145     "dart": "dart",
 146     "delphi": "delphi",
 147     "diff": "diff",
 148     "django": "django",
 149     "docker": "docker",
 150     "elixir": "elixir",
 151     "erlang": "erlang",
 152     "go": "go",
 153     "h": "c",
 154     "handlebars": "handlebars",
 155     "haskell": "haskell",
 156     "hs": "haskell",
 157     "html": "html",
 158     "ini": "ini",
 159     "ipy": "ipythonconsole",
 160     "ipynb": "ipythonconsole",
 161     "irc": "irc",
 162     "j2": "django",
 163     "java": "java",
 164     "js": "js",
 165     "json": "json",
 166     "jsx": "jsx",
 167     "kt": "kotlin",
 168     "less": "less",
 169     "lisp": "common-lisp",
 170     "lsp": "common-lisp",
 171     "lua": "lua",
 172     "m": "objective-c",
 173     "make": "make",
 174     "matlab": "matlab",
 175     "md": "_markdown",
 176     "nginx": "nginx",
 177     "numpy": "numpy",
 178     "patch": "diff",
 179     "perl": "perl",
 180     "php": "php",
 181     "pm": "perl",
 182     "postgresql": "postgresql",
 183     "py": "python",
 184     "rb": "rb",
 185     "rs": "rust",
 186     "rst": "rst",
 187     "sass": "sass",
 188     "scss": "scss",
 189     "sh": "bash",
 190     "sol": "sol",
 191     "sql": "sql",
 192     "swift": "swift",
 193     "tex": "tex",
 194     "typoscript": "typoscript",
 195     "vim": "vim",
 196     "xml": "xml",
 197     "xslt": "xslt",
 198     "yaml": "yaml",
 199     "yml": "yaml",
 200 }
 201
 202
 203 def guess_highlighter_from_path(path):
 204     """Return a known highlighter from a given path
 205
 206     Attempt to select a highlighter by checking the file extension in the mapping
 207     of extensions to highlighter. If that fails, attempt to pass the basename of
 208     the file. Return `_code` as the default highlighter if that fails.
 209     """
 210     import os
 211
 212     _name, ext = os.path.splitext(path)
 213
 214     if ext.startswith("."):
 215         ext = ext[1:]
 216
 217     if ext in EXTENSION_TO_HIGHLIGHTER:
 218         return EXTENSION_TO_HIGHLIGHTER[ext]
 219
 220     basename = os.path.basename(path)
 221
 222     return EXTENSION_TO_HIGHLIGHTER.get(basename, "_code")
 223
 224
 225 PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
 226
 227 PASTEMO_URL = "https://paste.mozilla.org/api/"
 228
 229 MACH_PASTEBIN_DESCRIPTION = """
 230 Command line interface to paste.mozilla.org.
 231
 232 Takes either a filename whose content should be pasted, or reads
 233 content from standard input. If a highlighter is specified it will
 234 be used, otherwise the file name will be used to determine an
 235 appropriate highlighter.
 236 """
 237
 238
 239 @CommandProvider
 240 class PastebinProvider(MachCommandBase):
 241     @Command("pastebin", category="misc", description=MACH_PASTEBIN_DESCRIPTION)
 242     @CommandArgument(
 243         "--list-highlighters",
 244         action="store_true",
 245         help="List known highlighters and exit",
 246     )
 247     @CommandArgument(
 248         "--highlighter", default=None, help="Syntax highlighting to use for paste"
 249     )
 250     @CommandArgument(
 251         "--expires",
 252         default="week",
 253         choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
 254         help="Expire paste after given time duration (default: %(default)s)",
 255     )
 256     @CommandArgument(
 257         "--verbose",
 258         action="store_true",
 259         help="Print extra info such as selected syntax highlighter",
 260     )
 261     @CommandArgument(
 262         "path",
 263         nargs="?",
 264         default=None,
 265         help="Path to file for upload to paste.mozilla.org",
 266     )
 267     def pastebin(
 268         self, command_context, list_highlighters, highlighter, expires, verbose, path
 269     ):
 270         import requests
 271
 272         def verbose_print(*args, **kwargs):
 273             """Print a string if `--verbose` flag is set"""
 274             if verbose:
 275                 print(*args, **kwargs)
 276
 277         # Show known highlighters and exit.
 278         if list_highlighters:
 279             lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
 280             print("Available lexers:\n" "    - %s" % "\n    - ".join(sorted(lexers)))
 281             return 0
 282
 283         # Get a correct expiry value.
 284         try:
 285             verbose_print("Setting expiry from %s" % expires)
 286             expires = MACH_PASTEBIN_DURATIONS[expires]
 287             verbose_print("Using %s as expiry" % expires)
 288         except KeyError:
 289             print(
 290                 "%s is not a valid duration.\n"
 291                 "(hint: try one of %s)"
 292                 % (expires, ", ".join(MACH_PASTEBIN_DURATIONS.keys()))
 293             )
 294             return 1
 295
 296         data = {
 297             "format": "json",
 298             "expires": expires,
 299         }
 300
 301         # Get content to be pasted.
 302         if path:
 303             verbose_print("Reading content from %s" % path)
 304             try:
 305                 with open(path, "r") as f:
 306                     content = f.read()
 307             except IOError:
 308                 print("ERROR. No such file %s" % path)
 309                 return 1
 310
 311             lexer = guess_highlighter_from_path(path)
 312             if lexer:
 313                 data["lexer"] = lexer
 314         else:
 315             verbose_print("Reading content from stdin")
 316             content = sys.stdin.read()
 317
 318         # Assert the length of content to be posted does not exceed the maximum.
 319         content_length = len(content)
 320         verbose_print("Checking size of content is okay (%d)" % content_length)
 321         if content_length > PASTEMO_MAX_CONTENT_LENGTH:
 322             print(
 323                 "Paste content is too large (%d, maximum %d)"
 324                 % (content_length, PASTEMO_MAX_CONTENT_LENGTH)
 325             )
 326             return 1
 327
 328         data["content"] = content
 329
 330         # Highlight as specified language, overwriting value set from filename.
 331         if highlighter:
 332             verbose_print("Setting %s as highlighter" % highlighter)
 333             data["lexer"] = highlighter
 334
 335         try:
 336             verbose_print("Sending request to %s" % PASTEMO_URL)
 337             resp = requests.post(PASTEMO_URL, data=data)
 338
 339             # Error code should always be 400.
 340             # Response content will include a helpful error message,
 341             # so print it here (for example, if an invalid highlighter is
 342             # provided, it will return a list of valid highlighters).
 343             if resp.status_code >= 400:
 344                 print("Error code %d: %s" % (resp.status_code, resp.content))
 345                 return 1
 346
 347             verbose_print("Pasted successfully")
 348
 349             response_json = resp.json()
 350
 351             verbose_print("Paste highlighted as %s" % response_json["lexer"])
 352             print(response_json["url"])
 353
 354             return 0
 355         except Exception as e:
 356             print("ERROR. Paste failed.")
 357             print("%s" % e)
 358         return 1
 359
 360
 361 class PypiBasedTool:
 362     """
 363     Helper for loading a tool that is hosted on pypi. The package is expected
 364     to expose a `mach_interface` module which has `new_release_on_pypi`,
 365     `parser`, and `run` functions.
 366     """
 367
 368     def __init__(self, module_name, pypi_name=None):
 369         self.name = module_name
 370         self.pypi_name = pypi_name or module_name
 371
 372     def _import(self):
 373         # Lazy loading of the tools mach interface.
 374         # Note that only the mach_interface module should be used from this file.
 375         import importlib
 376
 377         try:
 378             return importlib.import_module("%s.mach_interface" % self.name)
 379         except ImportError:
 380             return None
 381
 382     def create_parser(self, subcommand=None):
 383         # Create the command line parser.
 384         # If the tool is not installed, or not up to date, it will
 385         # first be installed.
 386         cmd = MozbuildObject.from_environment()
 387         cmd.activate_virtualenv()
 388         tool = self._import()
 389         if not tool:
 390             # The tool is not here at all, install it
 391             cmd.virtualenv_manager.install_pip_package(self.pypi_name)
 392             print(
 393                 "%s was installed. please re-run your"
 394                 " command. If you keep getting this message please "
 395                 " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name)
 396             )
 397         else:
 398             # Check if there is a new release available
 399             release = tool.new_release_on_pypi()
 400             if release:
 401                 print(release)
 402                 # there is one, so install it. Note that install_pip_package
 403                 # does not work here, so just run pip directly.
 404                 cmd.virtualenv_manager._run_pip(
 405                     ["install", "%s==%s" % (self.pypi_name, release)]
 406                 )
 407                 print(
 408                     "%s was updated to version %s. please"
 409                     " re-run your command." % (self.pypi_name, release)
 410                 )
 411             else:
 412                 # Tool is up to date, return the parser.
 413                 if subcommand:
 414                     return tool.parser(subcommand)
 415                 else:
 416                     return tool.parser()
 417         # exit if we updated or installed mozregression because
 418         # we may have already imported mozregression and running it
 419         # as this may cause issues.
 420         sys.exit(0)
 421
 422     def run(self, **options):
 423         tool = self._import()
 424         tool.run(options)
 425
 426
 427 def mozregression_create_parser():
 428     # Create the mozregression command line parser.
 429     # if mozregression is not installed, or not up to date, it will
 430     # first be installed.
 431     loader = PypiBasedTool("mozregression")
 432     return loader.create_parser()
 433
 434
 435 @CommandProvider
 436 class MozregressionCommand(MachCommandBase):
 437     @Command(
 438         "mozregression",
 439         category="misc",
 440         description=("Regression range finder for nightly" " and inbound builds."),
 441         parser=mozregression_create_parser,
 442     )
 443     def run(self, command_context, **options):
 444         command_context.activate_virtualenv()
 445         mozregression = PypiBasedTool("mozregression")
 446         mozregression.run(**options)
 447
 448
 449 @CommandProvider
 450 class NodeCommands(MachCommandBase):
 451     @Command(
 452         "node",
 453         category="devenv",
 454         description="Run the NodeJS interpreter used for building.",
 455     )
 456     @CommandArgument("args", nargs=argparse.REMAINDER)
 457     def node(self, command_context, args):
 458         from mozbuild.nodeutil import find_node_executable
 459
 460         # Avoid logging the command
 461         command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 462
 463         node_path, _ = find_node_executable()
 464
 465         return command_context.run_process(
 466             [node_path] + args,
 467             pass_thru=True,  # Allow user to run Node interactively.
 468             ensure_exit_code=False,  # Don't throw on non-zero exit code.
 469         )
 470
 471     @Command(
 472         "npm",
 473         category="devenv",
 474         description="Run the npm executable from the NodeJS used for building.",
 475     )
 476     @CommandArgument("args", nargs=argparse.REMAINDER)
 477     def npm(self, command_context, args):
 478         from mozbuild.nodeutil import find_npm_executable
 479
 480         # Avoid logging the command
 481         command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 482
 483         import os
 484
 485         # Add node and npm from mozbuild to front of system path
 486         #
 487         # This isn't pretty, but npm currently executes itself with
 488         # `#!/usr/bin/env node`, which means it just uses the node in the
 489         # current PATH. As a result, stuff gets built wrong and installed
 490         # in the wrong places and probably other badness too without this:
 491         npm_path, _ = find_npm_executable()
 492         if not npm_path:
 493             exit(-1, "could not find npm executable")
 494         path = os.path.abspath(os.path.dirname(npm_path))
 495         os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
 496
 497         return command_context.run_process(
 498             [npm_path, "--scripts-prepend-node-path=auto"] + args,
 499             pass_thru=True,  # Avoid eating npm output/error messages
 500             ensure_exit_code=False,  # Don't throw on non-zero exit code.
 501         )
 502
 503
 504 def logspam_create_parser(subcommand):
 505     # Create the logspam command line parser.
 506     # if logspam is not installed, or not up to date, it will
 507     # first be installed.
 508     loader = PypiBasedTool("logspam", "mozilla-log-spam")
 509     return loader.create_parser(subcommand)
 510
 511
 512 from functools import partial
 513
 514
 515 @CommandProvider
 516 class LogspamCommand(MachCommandBase):
 517     @Command(
 518         "logspam",
 519         category="misc",
 520         description=("Warning categorizer for treeherder test runs."),
 521     )
 522     def logspam(self, command_context):
 523         pass
 524
 525     @SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
 526     def report(self, command_context, **options):
 527         command_context.activate_virtualenv()
 528         logspam = PypiBasedTool("logspam")
 529         logspam.run(command="report", **options)
 530
 531     @SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
 532     def bisect(self, command_context, **options):
 533         command_context.activate_virtualenv()
 534         logspam = PypiBasedTool("logspam")
 535         logspam.run(command="bisect", **options)
 536
 537     @SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
 538     def create(self, command_context, **options):
 539         command_context.activate_virtualenv()
 540         logspam = PypiBasedTool("logspam")
 541         logspam.run(command="file", **options)