tools/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import logging
   7 import subprocess
   8 import sys
   9 from datetime import datetime, timedelta
  10 from operator import itemgetter
  11
  12 from mach.decorators import Command, CommandArgument, SubCommand
  13 from mozbuild.base import MozbuildObject
  14
  15
  16 def _get_busted_bugs(payload):
  17     import requests
  18
  19     payload = dict(payload)
  20     payload["include_fields"] = "id,summary,last_change_time,resolution"
  21     payload["blocks"] = 1543241
  22     response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload)
  23     response.raise_for_status()
  24     return response.json().get("bugs", [])
  25
  26
  27 @Command(
  28     "busted",
  29     category="misc",
  30     description="Query known bugs in our tooling, and file new ones.",
  31 )
  32 def busted_default(command_context):
  33     unresolved = _get_busted_bugs({"resolution": "---"})
  34     creation_time = datetime.now() - timedelta(days=15)
  35     creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ")
  36     resolved = _get_busted_bugs({"creation_time": creation_time})
  37     resolved = [bug for bug in resolved if bug["resolution"]]
  38     all_bugs = sorted(
  39         unresolved + resolved, key=itemgetter("last_change_time"), reverse=True
  40     )
  41     if all_bugs:
  42         for bug in all_bugs:
  43             print(
  44                 "[%s] Bug %s - %s"
  45                 % (
  46                     "UNRESOLVED"
  47                     if not bug["resolution"]
  48                     else "RESOLVED - %s" % bug["resolution"],
  49                     bug["id"],
  50                     bug["summary"],
  51                 )
  52             )
  53     else:
  54         print("No known tooling issues found.")
  55
  56
  57 @SubCommand("busted", "file", description="File a bug for busted tooling.")
  58 @CommandArgument(
  59     "against",
  60     help=(
  61         "The specific mach command that is busted (i.e. if you encountered "
  62         "an error with `mach build`, run `mach busted file build`). If "
  63         "the issue is not connected to any particular mach command, you "
  64         "can also run `mach busted file general`."
  65     ),
  66 )
  67 def busted_file(command_context, against):
  68     import webbrowser
  69
  70     if (
  71         against != "general"
  72         and against not in command_context._mach_context.commands.command_handlers
  73     ):
  74         print(
  75             "%s is not a valid value for `against`. `against` must be "
  76             "the name of a `mach` command, or else the string "
  77             '"general".' % against
  78         )
  79         return 1
  80
  81     if against == "general":
  82         product = "Firefox Build System"
  83         component = "General"
  84     else:
  85         import inspect
  86
  87         import mozpack.path as mozpath
  88
  89         # Look up the file implementing that command, then cross-refernce
  90         # moz.build files to get the product/component.
  91         handler = command_context._mach_context.commands.command_handlers[against]
  92         sourcefile = mozpath.relpath(
  93             inspect.getsourcefile(handler.func), command_context.topsrcdir
  94         )
  95         reader = command_context.mozbuild_reader(config_mode="empty")
  96         try:
  97             res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
  98             product, component = res.product, res.component
  99         except TypeError:
 100             # The file might not have a bug set.
 101             product = "Firefox Build System"
 102             component = "General"
 103
 104     uri = (
 105         "https://bugzilla.mozilla.org/enter_bug.cgi?"
 106         "product=%s&component=%s&blocked=1543241" % (product, component)
 107     )
 108     webbrowser.open_new_tab(uri)
 109
 110
 111 MACH_PASTEBIN_DURATIONS = {
 112     "onetime": "onetime",
 113     "hour": "3600",
 114     "day": "86400",
 115     "week": "604800",
 116     "month": "2073600",
 117 }
 118
 119 EXTENSION_TO_HIGHLIGHTER = {
 120     ".hgrc": "ini",
 121     "Dockerfile": "docker",
 122     "Makefile": "make",
 123     "applescript": "applescript",
 124     "arduino": "arduino",
 125     "bash": "bash",
 126     "bat": "bat",
 127     "c": "c",
 128     "clojure": "clojure",
 129     "cmake": "cmake",
 130     "coffee": "coffee-script",
 131     "console": "console",
 132     "cpp": "cpp",
 133     "cs": "csharp",
 134     "css": "css",
 135     "cu": "cuda",
 136     "cuda": "cuda",
 137     "dart": "dart",
 138     "delphi": "delphi",
 139     "diff": "diff",
 140     "django": "django",
 141     "docker": "docker",
 142     "elixir": "elixir",
 143     "erlang": "erlang",
 144     "go": "go",
 145     "h": "c",
 146     "handlebars": "handlebars",
 147     "haskell": "haskell",
 148     "hs": "haskell",
 149     "html": "html",
 150     "ini": "ini",
 151     "ipy": "ipythonconsole",
 152     "ipynb": "ipythonconsole",
 153     "irc": "irc",
 154     "j2": "django",
 155     "java": "java",
 156     "js": "js",
 157     "json": "json",
 158     "jsx": "jsx",
 159     "kt": "kotlin",
 160     "less": "less",
 161     "lisp": "common-lisp",
 162     "lsp": "common-lisp",
 163     "lua": "lua",
 164     "m": "objective-c",
 165     "make": "make",
 166     "matlab": "matlab",
 167     "md": "_markdown",
 168     "nginx": "nginx",
 169     "numpy": "numpy",
 170     "patch": "diff",
 171     "perl": "perl",
 172     "php": "php",
 173     "pm": "perl",
 174     "postgresql": "postgresql",
 175     "py": "python",
 176     "rb": "rb",
 177     "rs": "rust",
 178     "rst": "rst",
 179     "sass": "sass",
 180     "scss": "scss",
 181     "sh": "bash",
 182     "sol": "sol",
 183     "sql": "sql",
 184     "swift": "swift",
 185     "tex": "tex",
 186     "typoscript": "typoscript",
 187     "vim": "vim",
 188     "xml": "xml",
 189     "xslt": "xslt",
 190     "yaml": "yaml",
 191     "yml": "yaml",
 192 }
 193
 194
 195 def guess_highlighter_from_path(path):
 196     """Return a known highlighter from a given path
 197
 198     Attempt to select a highlighter by checking the file extension in the mapping
 199     of extensions to highlighter. If that fails, attempt to pass the basename of
 200     the file. Return `_code` as the default highlighter if that fails.
 201     """
 202     import os
 203
 204     _name, ext = os.path.splitext(path)
 205
 206     if ext.startswith("."):
 207         ext = ext[1:]
 208
 209     if ext in EXTENSION_TO_HIGHLIGHTER:
 210         return EXTENSION_TO_HIGHLIGHTER[ext]
 211
 212     basename = os.path.basename(path)
 213
 214     return EXTENSION_TO_HIGHLIGHTER.get(basename, "_code")
 215
 216
 217 PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
 218
 219 PASTEMO_URL = "https://paste.mozilla.org/api/"
 220
 221
 222 @Command(
 223     "pastebin",
 224     category="misc",
 225     description="Command line interface to paste.mozilla.org.",
 226 )
 227 @CommandArgument(
 228     "--list-highlighters",
 229     action="store_true",
 230     help="List known highlighters and exit",
 231 )
 232 @CommandArgument(
 233     "--highlighter", default=None, help="Syntax highlighting to use for paste"
 234 )
 235 @CommandArgument(
 236     "--expires",
 237     default="week",
 238     choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
 239     help="Expire paste after given time duration (default: %(default)s)",
 240 )
 241 @CommandArgument(
 242     "--verbose",
 243     action="store_true",
 244     help="Print extra info such as selected syntax highlighter",
 245 )
 246 @CommandArgument(
 247     "path",
 248     nargs="?",
 249     default=None,
 250     help="Path to file for upload to paste.mozilla.org",
 251 )
 252 def pastebin(command_context, list_highlighters, highlighter, expires, verbose, path):
 253     """Command line interface to `paste.mozilla.org`.
 254
 255     Takes either a filename whose content should be pasted, or reads
 256     content from standard input. If a highlighter is specified it will
 257     be used, otherwise the file name will be used to determine an
 258     appropriate highlighter.
 259     """
 260
 261     import requests
 262
 263     def verbose_print(*args, **kwargs):
 264         """Print a string if `--verbose` flag is set"""
 265         if verbose:
 266             print(*args, **kwargs)
 267
 268     # Show known highlighters and exit.
 269     if list_highlighters:
 270         lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
 271         print("Available lexers:\n    - %s" % "\n    - ".join(sorted(lexers)))
 272         return 0
 273
 274     # Get a correct expiry value.
 275     try:
 276         verbose_print("Setting expiry from %s" % expires)
 277         expires = MACH_PASTEBIN_DURATIONS[expires]
 278         verbose_print("Using %s as expiry" % expires)
 279     except KeyError:
 280         print(
 281             "%s is not a valid duration.\n"
 282             "(hint: try one of %s)"
 283             % (expires, ", ".join(MACH_PASTEBIN_DURATIONS.keys()))
 284         )
 285         return 1
 286
 287     data = {
 288         "format": "json",
 289         "expires": expires,
 290     }
 291
 292     # Get content to be pasted.
 293     if path:
 294         verbose_print("Reading content from %s" % path)
 295         try:
 296             with open(path, "r") as f:
 297                 content = f.read()
 298         except IOError:
 299             print("ERROR. No such file %s" % path)
 300             return 1
 301
 302         lexer = guess_highlighter_from_path(path)
 303         if lexer:
 304             data["lexer"] = lexer
 305     else:
 306         verbose_print("Reading content from stdin")
 307         content = sys.stdin.read()
 308
 309     # Assert the length of content to be posted does not exceed the maximum.
 310     content_length = len(content)
 311     verbose_print("Checking size of content is okay (%d)" % content_length)
 312     if content_length > PASTEMO_MAX_CONTENT_LENGTH:
 313         print(
 314             "Paste content is too large (%d, maximum %d)"
 315             % (content_length, PASTEMO_MAX_CONTENT_LENGTH)
 316         )
 317         return 1
 318
 319     data["content"] = content
 320
 321     # Highlight as specified language, overwriting value set from filename.
 322     if highlighter:
 323         verbose_print("Setting %s as highlighter" % highlighter)
 324         data["lexer"] = highlighter
 325
 326     try:
 327         verbose_print("Sending request to %s" % PASTEMO_URL)
 328         resp = requests.post(PASTEMO_URL, data=data)
 329
 330         # Error code should always be 400.
 331         # Response content will include a helpful error message,
 332         # so print it here (for example, if an invalid highlighter is
 333         # provided, it will return a list of valid highlighters).
 334         if resp.status_code >= 400:
 335             print("Error code %d: %s" % (resp.status_code, resp.content))
 336             return 1
 337
 338         verbose_print("Pasted successfully")
 339
 340         response_json = resp.json()
 341
 342         verbose_print("Paste highlighted as %s" % response_json["lexer"])
 343         print(response_json["url"])
 344
 345         return 0
 346     except Exception as e:
 347         print("ERROR. Paste failed.")
 348         print("%s" % e)
 349     return 1
 350
 351
 352 class PypiBasedTool:
 353     """
 354     Helper for loading a tool that is hosted on pypi. The package is expected
 355     to expose a `mach_interface` module which has `new_release_on_pypi`,
 356     `parser`, and `run` functions.
 357     """
 358
 359     def __init__(self, module_name, pypi_name=None):
 360         self.name = module_name
 361         self.pypi_name = pypi_name or module_name
 362
 363     def _import(self):
 364         # Lazy loading of the tools mach interface.
 365         # Note that only the mach_interface module should be used from this file.
 366         import importlib
 367
 368         try:
 369             return importlib.import_module("%s.mach_interface" % self.name)
 370         except ImportError:
 371             return None
 372
 373     def create_parser(self, subcommand=None):
 374         # Create the command line parser.
 375         # If the tool is not installed, or not up to date, it will
 376         # first be installed.
 377         cmd = MozbuildObject.from_environment()
 378         cmd.activate_virtualenv()
 379         tool = self._import()
 380         if not tool:
 381             # The tool is not here at all, install it
 382             cmd.virtualenv_manager.install_pip_package(self.pypi_name)
 383             print(
 384                 "%s was installed. please re-run your"
 385                 " command. If you keep getting this message please "
 386                 " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name)
 387             )
 388         else:
 389             # Check if there is a new release available
 390             release = tool.new_release_on_pypi()
 391             if release:
 392                 print(release)
 393                 # there is one, so install it. Note that install_pip_package
 394                 # does not work here, so just run pip directly.
 395                 subprocess.check_call(
 396                     [
 397                         cmd.virtualenv_manager.python_path,
 398                         "-m",
 399                         "pip",
 400                         "install",
 401                         f"{self.pypi_name}=={release}",
 402                     ]
 403                 )
 404                 print(
 405                     "%s was updated to version %s. please"
 406                     " re-run your command." % (self.pypi_name, release)
 407                 )
 408             else:
 409                 # Tool is up to date, return the parser.
 410                 if subcommand:
 411                     return tool.parser(subcommand)
 412                 else:
 413                     return tool.parser()
 414         # exit if we updated or installed mozregression because
 415         # we may have already imported mozregression and running it
 416         # as this may cause issues.
 417         sys.exit(0)
 418
 419     def run(self, **options):
 420         tool = self._import()
 421         tool.run(options)
 422
 423
 424 def mozregression_create_parser():
 425     # Create the mozregression command line parser.
 426     # if mozregression is not installed, or not up to date, it will
 427     # first be installed.
 428     loader = PypiBasedTool("mozregression")
 429     return loader.create_parser()
 430
 431
 432 @Command(
 433     "mozregression",
 434     category="misc",
 435     description="Regression range finder for nightly and inbound builds.",
 436     parser=mozregression_create_parser,
 437 )
 438 def run(command_context, **options):
 439     command_context.activate_virtualenv()
 440     mozregression = PypiBasedTool("mozregression")
 441     mozregression.run(**options)
 442
 443
 444 @Command(
 445     "node",
 446     category="devenv",
 447     description="Run the NodeJS interpreter used for building.",
 448 )
 449 @CommandArgument("args", nargs=argparse.REMAINDER)
 450 def node(command_context, args):
 451     from mozbuild.nodeutil import find_node_executable
 452
 453     # Avoid logging the command
 454     command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 455
 456     node_path, _ = find_node_executable()
 457
 458     return command_context.run_process(
 459         [node_path] + args,
 460         pass_thru=True,  # Allow user to run Node interactively.
 461         ensure_exit_code=False,  # Don't throw on non-zero exit code.
 462     )
 463
 464
 465 @Command(
 466     "npm",
 467     category="devenv",
 468     description="Run the npm executable from the NodeJS used for building.",
 469 )
 470 @CommandArgument("args", nargs=argparse.REMAINDER)
 471 def npm(command_context, args):
 472     from mozbuild.nodeutil import find_npm_executable
 473
 474     # Avoid logging the command
 475     command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 476
 477     import os
 478
 479     # Add node and npm from mozbuild to front of system path
 480     #
 481     # This isn't pretty, but npm currently executes itself with
 482     # `#!/usr/bin/env node`, which means it just uses the node in the
 483     # current PATH. As a result, stuff gets built wrong and installed
 484     # in the wrong places and probably other badness too without this:
 485     npm_path, _ = find_npm_executable()
 486     if not npm_path:
 487         exit(-1, "could not find npm executable")
 488     path = os.path.abspath(os.path.dirname(npm_path))
 489     os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
 490
 491     # karma-firefox-launcher needs the path to firefox binary.
 492     firefox_bin = command_context.get_binary_path(validate_exists=False)
 493     if os.path.exists(firefox_bin):
 494         os.environ["FIREFOX_BIN"] = firefox_bin
 495
 496     return command_context.run_process(
 497         [npm_path, "--scripts-prepend-node-path=auto"] + args,
 498         pass_thru=True,  # Avoid eating npm output/error messages
 499         ensure_exit_code=False,  # Don't throw on non-zero exit code.
 500     )
 501
 502
 503 def logspam_create_parser(subcommand):
 504     # Create the logspam command line parser.
 505     # if logspam is not installed, or not up to date, it will
 506     # first be installed.
 507     loader = PypiBasedTool("logspam", "mozilla-log-spam")
 508     return loader.create_parser(subcommand)
 509
 510
 511 from functools import partial
 512
 513
 514 @Command(
 515     "logspam",
 516     category="misc",
 517     description="Warning categorizer for treeherder test runs.",
 518 )
 519 def logspam(command_context):
 520     pass
 521
 522
 523 @SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
 524 def report(command_context, **options):
 525     command_context.activate_virtualenv()
 526     logspam = PypiBasedTool("logspam")
 527     logspam.run(command="report", **options)
 528
 529
 530 @SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
 531 def bisect(command_context, **options):
 532     command_context.activate_virtualenv()
 533     logspam = PypiBasedTool("logspam")
 534     logspam.run(command="bisect", **options)
 535
 536
 537 @SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
 538 def create(command_context, **options):
 539     command_context.activate_virtualenv()
 540     logspam = PypiBasedTool("logspam")
 541     logspam.run(command="file", **options)
 542
 543
 544 # mots_loader will be used when running commands and subcommands, as well as
 545 # when creating the parsers.
 546 mots_loader = PypiBasedTool("mots")
 547
 548
 549 def mots_create_parser(subcommand=None):
 550     return mots_loader.create_parser(subcommand)
 551
 552
 553 def mots_run_subcommand(command, command_context, **options):
 554     command_context.activate_virtualenv()
 555     mots_loader.run(command=command, **options)
 556
 557
 558 class motsSubCommand(SubCommand):
 559     """A helper subclass that reduces repitition when defining subcommands."""
 560
 561     def __init__(self, subcommand):
 562         super().__init__(
 563             "mots",
 564             subcommand,
 565             parser=partial(mots_create_parser, subcommand),
 566         )
 567
 568
 569 @Command(
 570     "mots",
 571     category="misc",
 572     description="Manage module information in-tree using the mots CLI.",
 573     parser=mots_create_parser,
 574 )
 575 def mots(command_context, **options):
 576     """The main mots command call."""
 577     command_context.activate_virtualenv()
 578     mots_loader.run(**options)
 579
 580
 581 # Define subcommands that will be proxied through mach.
 582 for sc in (
 583     "clean",
 584     "check-hashes",
 585     "export",
 586     "export-and-clean",
 587     "module",
 588     "query",
 589     "settings",
 590     "user",
 591     "validate",
 592 ):
 593     # Pass through args and kwargs, but add the subcommand string as the first argument.
 594     motsSubCommand(sc)(lambda *a, **kw: mots_run_subcommand(sc, *a, **kw))