tools/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import argparse
   8 import subprocess
   9 from datetime import datetime, timedelta
  10 import logging
  11 from operator import itemgetter
  12 import sys
  13
  14 from mach.decorators import (
  15     CommandArgument,
  16     Command,
  17     SubCommand,
  18 )
  19
  20 from mozbuild.base import MozbuildObject
  21
  22
  23 def _get_busted_bugs(payload):
  24     import requests
  25
  26     payload = dict(payload)
  27     payload["include_fields"] = "id,summary,last_change_time,resolution"
  28     payload["blocks"] = 1543241
  29     response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload)
  30     response.raise_for_status()
  31     return response.json().get("bugs", [])
  32
  33
  34 @Command(
  35     "busted",
  36     category="misc",
  37     description="Query known bugs in our tooling, and file new ones.",
  38 )
  39 def busted_default(command_context):
  40     unresolved = _get_busted_bugs({"resolution": "---"})
  41     creation_time = datetime.now() - timedelta(days=15)
  42     creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ")
  43     resolved = _get_busted_bugs({"creation_time": creation_time})
  44     resolved = [bug for bug in resolved if bug["resolution"]]
  45     all_bugs = sorted(
  46         unresolved + resolved, key=itemgetter("last_change_time"), reverse=True
  47     )
  48     if all_bugs:
  49         for bug in all_bugs:
  50             print(
  51                 "[%s] Bug %s - %s"
  52                 % (
  53                     "UNRESOLVED"
  54                     if not bug["resolution"]
  55                     else "RESOLVED - %s" % bug["resolution"],
  56                     bug["id"],
  57                     bug["summary"],
  58                 )
  59             )
  60     else:
  61         print("No known tooling issues found.")
  62
  63
  64 @SubCommand("busted", "file", description="File a bug for busted tooling.")
  65 @CommandArgument(
  66     "against",
  67     help=(
  68         "The specific mach command that is busted (i.e. if you encountered "
  69         "an error with `mach build`, run `mach busted file build`). If "
  70         "the issue is not connected to any particular mach command, you "
  71         "can also run `mach busted file general`."
  72     ),
  73 )
  74 def busted_file(command_context, against):
  75     import webbrowser
  76
  77     if (
  78         against != "general"
  79         and against not in command_context._mach_context.commands.command_handlers
  80     ):
  81         print(
  82             "%s is not a valid value for `against`. `against` must be "
  83             "the name of a `mach` command, or else the string "
  84             '"general".' % against
  85         )
  86         return 1
  87
  88     if against == "general":
  89         product = "Firefox Build System"
  90         component = "General"
  91     else:
  92         import inspect
  93         import mozpack.path as mozpath
  94
  95         # Look up the file implementing that command, then cross-refernce
  96         # moz.build files to get the product/component.
  97         handler = command_context._mach_context.commands.command_handlers[against]
  98         sourcefile = mozpath.relpath(
  99             inspect.getsourcefile(handler.func), command_context.topsrcdir
 100         )
 101         reader = command_context.mozbuild_reader(config_mode="empty")
 102         try:
 103             res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
 104             product, component = res.product, res.component
 105         except TypeError:
 106             # The file might not have a bug set.
 107             product = "Firefox Build System"
 108             component = "General"
 109
 110     uri = (
 111         "https://bugzilla.mozilla.org/enter_bug.cgi?"
 112         "product=%s&component=%s&blocked=1543241" % (product, component)
 113     )
 114     webbrowser.open_new_tab(uri)
 115
 116
 117 MACH_PASTEBIN_DURATIONS = {
 118     "onetime": "onetime",
 119     "hour": "3600",
 120     "day": "86400",
 121     "week": "604800",
 122     "month": "2073600",
 123 }
 124
 125 EXTENSION_TO_HIGHLIGHTER = {
 126     ".hgrc": "ini",
 127     "Dockerfile": "docker",
 128     "Makefile": "make",
 129     "applescript": "applescript",
 130     "arduino": "arduino",
 131     "bash": "bash",
 132     "bat": "bat",
 133     "c": "c",
 134     "clojure": "clojure",
 135     "cmake": "cmake",
 136     "coffee": "coffee-script",
 137     "console": "console",
 138     "cpp": "cpp",
 139     "cs": "csharp",
 140     "css": "css",
 141     "cu": "cuda",
 142     "cuda": "cuda",
 143     "dart": "dart",
 144     "delphi": "delphi",
 145     "diff": "diff",
 146     "django": "django",
 147     "docker": "docker",
 148     "elixir": "elixir",
 149     "erlang": "erlang",
 150     "go": "go",
 151     "h": "c",
 152     "handlebars": "handlebars",
 153     "haskell": "haskell",
 154     "hs": "haskell",
 155     "html": "html",
 156     "ini": "ini",
 157     "ipy": "ipythonconsole",
 158     "ipynb": "ipythonconsole",
 159     "irc": "irc",
 160     "j2": "django",
 161     "java": "java",
 162     "js": "js",
 163     "json": "json",
 164     "jsx": "jsx",
 165     "kt": "kotlin",
 166     "less": "less",
 167     "lisp": "common-lisp",
 168     "lsp": "common-lisp",
 169     "lua": "lua",
 170     "m": "objective-c",
 171     "make": "make",
 172     "matlab": "matlab",
 173     "md": "_markdown",
 174     "nginx": "nginx",
 175     "numpy": "numpy",
 176     "patch": "diff",
 177     "perl": "perl",
 178     "php": "php",
 179     "pm": "perl",
 180     "postgresql": "postgresql",
 181     "py": "python",
 182     "rb": "rb",
 183     "rs": "rust",
 184     "rst": "rst",
 185     "sass": "sass",
 186     "scss": "scss",
 187     "sh": "bash",
 188     "sol": "sol",
 189     "sql": "sql",
 190     "swift": "swift",
 191     "tex": "tex",
 192     "typoscript": "typoscript",
 193     "vim": "vim",
 194     "xml": "xml",
 195     "xslt": "xslt",
 196     "yaml": "yaml",
 197     "yml": "yaml",
 198 }
 199
 200
 201 def guess_highlighter_from_path(path):
 202     """Return a known highlighter from a given path
 203
 204     Attempt to select a highlighter by checking the file extension in the mapping
 205     of extensions to highlighter. If that fails, attempt to pass the basename of
 206     the file. Return `_code` as the default highlighter if that fails.
 207     """
 208     import os
 209
 210     _name, ext = os.path.splitext(path)
 211
 212     if ext.startswith("."):
 213         ext = ext[1:]
 214
 215     if ext in EXTENSION_TO_HIGHLIGHTER:
 216         return EXTENSION_TO_HIGHLIGHTER[ext]
 217
 218     basename = os.path.basename(path)
 219
 220     return EXTENSION_TO_HIGHLIGHTER.get(basename, "_code")
 221
 222
 223 PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
 224
 225 PASTEMO_URL = "https://paste.mozilla.org/api/"
 226
 227 MACH_PASTEBIN_DESCRIPTION = """
 228 Command line interface to paste.mozilla.org.
 229
 230 Takes either a filename whose content should be pasted, or reads
 231 content from standard input. If a highlighter is specified it will
 232 be used, otherwise the file name will be used to determine an
 233 appropriate highlighter.
 234 """
 235
 236
 237 @Command("pastebin", category="misc", description=MACH_PASTEBIN_DESCRIPTION)
 238 @CommandArgument(
 239     "--list-highlighters",
 240     action="store_true",
 241     help="List known highlighters and exit",
 242 )
 243 @CommandArgument(
 244     "--highlighter", default=None, help="Syntax highlighting to use for paste"
 245 )
 246 @CommandArgument(
 247     "--expires",
 248     default="week",
 249     choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
 250     help="Expire paste after given time duration (default: %(default)s)",
 251 )
 252 @CommandArgument(
 253     "--verbose",
 254     action="store_true",
 255     help="Print extra info such as selected syntax highlighter",
 256 )
 257 @CommandArgument(
 258     "path",
 259     nargs="?",
 260     default=None,
 261     help="Path to file for upload to paste.mozilla.org",
 262 )
 263 def pastebin(command_context, list_highlighters, highlighter, expires, verbose, path):
 264     import requests
 265
 266     def verbose_print(*args, **kwargs):
 267         """Print a string if `--verbose` flag is set"""
 268         if verbose:
 269             print(*args, **kwargs)
 270
 271     # Show known highlighters and exit.
 272     if list_highlighters:
 273         lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
 274         print("Available lexers:\n    - %s" % "\n    - ".join(sorted(lexers)))
 275         return 0
 276
 277     # Get a correct expiry value.
 278     try:
 279         verbose_print("Setting expiry from %s" % expires)
 280         expires = MACH_PASTEBIN_DURATIONS[expires]
 281         verbose_print("Using %s as expiry" % expires)
 282     except KeyError:
 283         print(
 284             "%s is not a valid duration.\n"
 285             "(hint: try one of %s)"
 286             % (expires, ", ".join(MACH_PASTEBIN_DURATIONS.keys()))
 287         )
 288         return 1
 289
 290     data = {
 291         "format": "json",
 292         "expires": expires,
 293     }
 294
 295     # Get content to be pasted.
 296     if path:
 297         verbose_print("Reading content from %s" % path)
 298         try:
 299             with open(path, "r") as f:
 300                 content = f.read()
 301         except IOError:
 302             print("ERROR. No such file %s" % path)
 303             return 1
 304
 305         lexer = guess_highlighter_from_path(path)
 306         if lexer:
 307             data["lexer"] = lexer
 308     else:
 309         verbose_print("Reading content from stdin")
 310         content = sys.stdin.read()
 311
 312     # Assert the length of content to be posted does not exceed the maximum.
 313     content_length = len(content)
 314     verbose_print("Checking size of content is okay (%d)" % content_length)
 315     if content_length > PASTEMO_MAX_CONTENT_LENGTH:
 316         print(
 317             "Paste content is too large (%d, maximum %d)"
 318             % (content_length, PASTEMO_MAX_CONTENT_LENGTH)
 319         )
 320         return 1
 321
 322     data["content"] = content
 323
 324     # Highlight as specified language, overwriting value set from filename.
 325     if highlighter:
 326         verbose_print("Setting %s as highlighter" % highlighter)
 327         data["lexer"] = highlighter
 328
 329     try:
 330         verbose_print("Sending request to %s" % PASTEMO_URL)
 331         resp = requests.post(PASTEMO_URL, data=data)
 332
 333         # Error code should always be 400.
 334         # Response content will include a helpful error message,
 335         # so print it here (for example, if an invalid highlighter is
 336         # provided, it will return a list of valid highlighters).
 337         if resp.status_code >= 400:
 338             print("Error code %d: %s" % (resp.status_code, resp.content))
 339             return 1
 340
 341         verbose_print("Pasted successfully")
 342
 343         response_json = resp.json()
 344
 345         verbose_print("Paste highlighted as %s" % response_json["lexer"])
 346         print(response_json["url"])
 347
 348         return 0
 349     except Exception as e:
 350         print("ERROR. Paste failed.")
 351         print("%s" % e)
 352     return 1
 353
 354
 355 class PypiBasedTool:
 356     """
 357     Helper for loading a tool that is hosted on pypi. The package is expected
 358     to expose a `mach_interface` module which has `new_release_on_pypi`,
 359     `parser`, and `run` functions.
 360     """
 361
 362     def __init__(self, module_name, pypi_name=None):
 363         self.name = module_name
 364         self.pypi_name = pypi_name or module_name
 365
 366     def _import(self):
 367         # Lazy loading of the tools mach interface.
 368         # Note that only the mach_interface module should be used from this file.
 369         import importlib
 370
 371         try:
 372             return importlib.import_module("%s.mach_interface" % self.name)
 373         except ImportError:
 374             return None
 375
 376     def create_parser(self, subcommand=None):
 377         # Create the command line parser.
 378         # If the tool is not installed, or not up to date, it will
 379         # first be installed.
 380         cmd = MozbuildObject.from_environment()
 381         cmd.activate_virtualenv()
 382         tool = self._import()
 383         if not tool:
 384             # The tool is not here at all, install it
 385             cmd.virtualenv_manager.install_pip_package(self.pypi_name)
 386             print(
 387                 "%s was installed. please re-run your"
 388                 " command. If you keep getting this message please "
 389                 " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name)
 390             )
 391         else:
 392             # Check if there is a new release available
 393             release = tool.new_release_on_pypi()
 394             if release:
 395                 print(release)
 396                 # there is one, so install it. Note that install_pip_package
 397                 # does not work here, so just run pip directly.
 398                 subprocess.check_call(
 399                     [
 400                         cmd.virtualenv_manager.python_path,
 401                         "-m",
 402                         "pip",
 403                         "install",
 404                         f"{self.pypi_name}=={release}",
 405                     ]
 406                 )
 407                 print(
 408                     "%s was updated to version %s. please"
 409                     " re-run your command." % (self.pypi_name, release)
 410                 )
 411             else:
 412                 # Tool is up to date, return the parser.
 413                 if subcommand:
 414                     return tool.parser(subcommand)
 415                 else:
 416                     return tool.parser()
 417         # exit if we updated or installed mozregression because
 418         # we may have already imported mozregression and running it
 419         # as this may cause issues.
 420         sys.exit(0)
 421
 422     def run(self, **options):
 423         tool = self._import()
 424         tool.run(options)
 425
 426
 427 def mozregression_create_parser():
 428     # Create the mozregression command line parser.
 429     # if mozregression is not installed, or not up to date, it will
 430     # first be installed.
 431     loader = PypiBasedTool("mozregression")
 432     return loader.create_parser()
 433
 434
 435 @Command(
 436     "mozregression",
 437     category="misc",
 438     description=("Regression range finder for nightly and inbound builds."),
 439     parser=mozregression_create_parser,
 440 )
 441 def run(command_context, **options):
 442     command_context.activate_virtualenv()
 443     mozregression = PypiBasedTool("mozregression")
 444     mozregression.run(**options)
 445
 446
 447 @Command(
 448     "node",
 449     category="devenv",
 450     description="Run the NodeJS interpreter used for building.",
 451 )
 452 @CommandArgument("args", nargs=argparse.REMAINDER)
 453 def node(command_context, args):
 454     from mozbuild.nodeutil import find_node_executable
 455
 456     # Avoid logging the command
 457     command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 458
 459     node_path, _ = find_node_executable()
 460
 461     return command_context.run_process(
 462         [node_path] + args,
 463         pass_thru=True,  # Allow user to run Node interactively.
 464         ensure_exit_code=False,  # Don't throw on non-zero exit code.
 465     )
 466
 467
 468 @Command(
 469     "npm",
 470     category="devenv",
 471     description="Run the npm executable from the NodeJS used for building.",
 472 )
 473 @CommandArgument("args", nargs=argparse.REMAINDER)
 474 def npm(command_context, args):
 475     from mozbuild.nodeutil import find_npm_executable
 476
 477     # Avoid logging the command
 478     command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 479
 480     import os
 481
 482     # Add node and npm from mozbuild to front of system path
 483     #
 484     # This isn't pretty, but npm currently executes itself with
 485     # `#!/usr/bin/env node`, which means it just uses the node in the
 486     # current PATH. As a result, stuff gets built wrong and installed
 487     # in the wrong places and probably other badness too without this:
 488     npm_path, _ = find_npm_executable()
 489     if not npm_path:
 490         exit(-1, "could not find npm executable")
 491     path = os.path.abspath(os.path.dirname(npm_path))
 492     os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
 493
 494     # karma-firefox-launcher needs the path to firefox binary.
 495     firefox_bin = command_context.get_binary_path(validate_exists=False)
 496     if os.path.exists(firefox_bin):
 497         os.environ["FIREFOX_BIN"] = firefox_bin
 498
 499     return command_context.run_process(
 500         [npm_path, "--scripts-prepend-node-path=auto"] + args,
 501         pass_thru=True,  # Avoid eating npm output/error messages
 502         ensure_exit_code=False,  # Don't throw on non-zero exit code.
 503     )
 504
 505
 506 def logspam_create_parser(subcommand):
 507     # Create the logspam command line parser.
 508     # if logspam is not installed, or not up to date, it will
 509     # first be installed.
 510     loader = PypiBasedTool("logspam", "mozilla-log-spam")
 511     return loader.create_parser(subcommand)
 512
 513
 514 from functools import partial
 515
 516
 517 @Command(
 518     "logspam",
 519     category="misc",
 520     description=("Warning categorizer for treeherder test runs."),
 521 )
 522 def logspam(command_context):
 523     pass
 524
 525
 526 @SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
 527 def report(command_context, **options):
 528     command_context.activate_virtualenv()
 529     logspam = PypiBasedTool("logspam")
 530     logspam.run(command="report", **options)
 531
 532
 533 @SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
 534 def bisect(command_context, **options):
 535     command_context.activate_virtualenv()
 536     logspam = PypiBasedTool("logspam")
 537     logspam.run(command="bisect", **options)
 538
 539
 540 @SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
 541 def create(command_context, **options):
 542     command_context.activate_virtualenv()
 543     logspam = PypiBasedTool("logspam")
 544     logspam.run(command="file", **options)