1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import absolute_import
, print_function
, unicode_literals
8 from datetime
import datetime
, timedelta
10 from operator
import itemgetter
13 from mach
.decorators
import (
20 from mozbuild
.base
import MachCommandBase
, MozbuildObject
23 def _get_busted_bugs(payload
):
26 payload
= dict(payload
)
27 payload
["include_fields"] = "id,summary,last_change_time,resolution"
28 payload
["blocks"] = 1543241
29 response
= requests
.get("https://bugzilla.mozilla.org/rest/bug", payload
)
30 response
.raise_for_status()
31 return response
.json().get("bugs", [])
35 class BustedProvider(MachCommandBase
):
39 description
="Query known bugs in our tooling, and file new ones.",
41 def busted_default(self
, command_context
):
42 unresolved
= _get_busted_bugs({"resolution": "---"})
43 creation_time
= datetime
.now() - timedelta(days
=15)
44 creation_time
= creation_time
.strftime("%Y-%m-%dT%H-%M-%SZ")
45 resolved
= _get_busted_bugs({"creation_time": creation_time
})
46 resolved
= [bug
for bug
in resolved
if bug
["resolution"]]
48 unresolved
+ resolved
, key
=itemgetter("last_change_time"), reverse
=True
56 if not bug
["resolution"]
57 else "RESOLVED - %s" % bug
["resolution"],
63 print("No known tooling issues found.")
65 @SubCommand("busted", "file", description
="File a bug for busted tooling.")
69 "The specific mach command that is busted (i.e. if you encountered "
70 "an error with `mach build`, run `mach busted file build`). If "
71 "the issue is not connected to any particular mach command, you "
72 "can also run `mach busted file general`."
75 def busted_file(self
, command_context
, against
):
80 and against
not in self
._mach
_context
.commands
.command_handlers
83 "%s is not a valid value for `against`. `against` must be "
84 "the name of a `mach` command, or else the string "
85 '"general".' % against
89 if against
== "general":
90 product
= "Firefox Build System"
94 import mozpack
.path
as mozpath
96 # Look up the file implementing that command, then cross-refernce
97 # moz.build files to get the product/component.
98 handler
= self
._mach
_context
.commands
.command_handlers
[against
]
99 method
= getattr(handler
.cls
, handler
.method
)
100 sourcefile
= mozpath
.relpath(inspect
.getsourcefile(method
), self
.topsrcdir
)
101 reader
= self
.mozbuild_reader(config_mode
="empty")
103 res
= reader
.files_info([sourcefile
])[sourcefile
]["BUG_COMPONENT"]
104 product
, component
= res
.product
, res
.component
106 # The file might not have a bug set.
107 product
= "Firefox Build System"
108 component
= "General"
111 "https://bugzilla.mozilla.org/enter_bug.cgi?"
112 "product=%s&component=%s&blocked=1543241" % (product
, component
)
114 webbrowser
.open_new_tab(uri
)
117 MACH_PASTEBIN_DURATIONS
= {
118 "onetime": "onetime",
125 EXTENSION_TO_HIGHLIGHTER
= {
127 "Dockerfile": "docker",
129 "applescript": "applescript",
130 "arduino": "arduino",
134 "clojure": "clojure",
136 "coffee": "coffee-script",
137 "console": "console",
152 "handlebars": "handlebars",
153 "haskell": "haskell",
157 "ipy": "ipythonconsole",
158 "ipynb": "ipythonconsole",
167 "lisp": "common-lisp",
168 "lsp": "common-lisp",
180 "postgresql": "postgresql",
192 "typoscript": "typoscript",
201 def guess_highlighter_from_path(path
):
202 """Return a known highlighter from a given path
204 Attempt to select a highlighter by checking the file extension in the mapping
205 of extensions to highlighter. If that fails, attempt to pass the basename of
206 the file. Return `_code` as the default highlighter if that fails.
210 _name
, ext
= os
.path
.splitext(path
)
212 if ext
.startswith("."):
215 if ext
in EXTENSION_TO_HIGHLIGHTER
:
216 return EXTENSION_TO_HIGHLIGHTER
[ext
]
218 basename
= os
.path
.basename(path
)
220 return EXTENSION_TO_HIGHLIGHTER
.get(basename
, "_code")
223 PASTEMO_MAX_CONTENT_LENGTH
= 250 * 1024 * 1024
225 PASTEMO_URL
= "https://paste.mozilla.org/api/"
227 MACH_PASTEBIN_DESCRIPTION
= """
228 Command line interface to paste.mozilla.org.
230 Takes either a filename whose content should be pasted, or reads
231 content from standard input. If a highlighter is specified it will
232 be used, otherwise the file name will be used to determine an
233 appropriate highlighter.
238 class PastebinProvider(MachCommandBase
):
239 @Command("pastebin", category
="misc", description
=MACH_PASTEBIN_DESCRIPTION
)
241 "--list-highlighters",
243 help="List known highlighters and exit",
246 "--highlighter", default
=None, help="Syntax highlighting to use for paste"
251 choices
=sorted(MACH_PASTEBIN_DURATIONS
.keys()),
252 help="Expire paste after given time duration (default: %(default)s)",
257 help="Print extra info such as selected syntax highlighter",
263 help="Path to file for upload to paste.mozilla.org",
266 self
, command_context
, list_highlighters
, highlighter
, expires
, verbose
, path
270 def verbose_print(*args
, **kwargs
):
271 """Print a string if `--verbose` flag is set"""
273 print(*args
, **kwargs
)
275 # Show known highlighters and exit.
276 if list_highlighters
:
277 lexers
= set(EXTENSION_TO_HIGHLIGHTER
.values())
278 print("Available lexers:\n" " - %s" % "\n - ".join(sorted(lexers
)))
281 # Get a correct expiry value.
283 verbose_print("Setting expiry from %s" % expires
)
284 expires
= MACH_PASTEBIN_DURATIONS
[expires
]
285 verbose_print("Using %s as expiry" % expires
)
288 "%s is not a valid duration.\n"
289 "(hint: try one of %s)"
290 % (expires
, ", ".join(MACH_PASTEBIN_DURATIONS
.keys()))
299 # Get content to be pasted.
301 verbose_print("Reading content from %s" % path
)
303 with
open(path
, "r") as f
:
306 print("ERROR. No such file %s" % path
)
309 lexer
= guess_highlighter_from_path(path
)
311 data
["lexer"] = lexer
313 verbose_print("Reading content from stdin")
314 content
= sys
.stdin
.read()
316 # Assert the length of content to be posted does not exceed the maximum.
317 content_length
= len(content
)
318 verbose_print("Checking size of content is okay (%d)" % content_length
)
319 if content_length
> PASTEMO_MAX_CONTENT_LENGTH
:
321 "Paste content is too large (%d, maximum %d)"
322 % (content_length
, PASTEMO_MAX_CONTENT_LENGTH
)
326 data
["content"] = content
328 # Highlight as specified language, overwriting value set from filename.
330 verbose_print("Setting %s as highlighter" % highlighter
)
331 data
["lexer"] = highlighter
334 verbose_print("Sending request to %s" % PASTEMO_URL
)
335 resp
= requests
.post(PASTEMO_URL
, data
=data
)
337 # Error code should always be 400.
338 # Response content will include a helpful error message,
339 # so print it here (for example, if an invalid highlighter is
340 # provided, it will return a list of valid highlighters).
341 if resp
.status_code
>= 400:
342 print("Error code %d: %s" % (resp
.status_code
, resp
.content
))
345 verbose_print("Pasted successfully")
347 response_json
= resp
.json()
349 verbose_print("Paste highlighted as %s" % response_json
["lexer"])
350 print(response_json
["url"])
353 except Exception as e
:
354 print("ERROR. Paste failed.")
361 Helper for loading a tool that is hosted on pypi. The package is expected
362 to expose a `mach_interface` module which has `new_release_on_pypi`,
363 `parser`, and `run` functions.
366 def __init__(self
, module_name
, pypi_name
=None):
367 self
.name
= module_name
368 self
.pypi_name
= pypi_name
or module_name
371 # Lazy loading of the tools mach interface.
372 # Note that only the mach_interface module should be used from this file.
376 return importlib
.import_module("%s.mach_interface" % self
.name
)
380 def create_parser(self
, subcommand
=None):
381 # Create the command line parser.
382 # If the tool is not installed, or not up to date, it will
383 # first be installed.
384 cmd
= MozbuildObject
.from_environment()
385 cmd
.activate_virtualenv()
386 tool
= self
._import
()
388 # The tool is not here at all, install it
389 cmd
.virtualenv_manager
.install_pip_package(self
.pypi_name
)
391 "%s was installed. please re-run your"
392 " command. If you keep getting this message please "
393 " manually run: 'pip install -U %s'." % (self
.pypi_name
, self
.pypi_name
)
396 # Check if there is a new release available
397 release
= tool
.new_release_on_pypi()
400 # there is one, so install it. Note that install_pip_package
401 # does not work here, so just run pip directly.
402 cmd
.virtualenv_manager
._run
_pip
(
403 ["install", "%s==%s" % (self
.pypi_name
, release
)]
406 "%s was updated to version %s. please"
407 " re-run your command." % (self
.pypi_name
, release
)
410 # Tool is up to date, return the parser.
412 return tool
.parser(subcommand
)
415 # exit if we updated or installed mozregression because
416 # we may have already imported mozregression and running it
417 # as this may cause issues.
420 def run(self
, **options
):
421 tool
= self
._import
()
425 def mozregression_create_parser():
426 # Create the mozregression command line parser.
427 # if mozregression is not installed, or not up to date, it will
428 # first be installed.
429 loader
= PypiBasedTool("mozregression")
430 return loader
.create_parser()
434 class MozregressionCommand(MachCommandBase
):
438 description
=("Regression range finder for nightly" " and inbound builds."),
439 parser
=mozregression_create_parser
,
441 def run(self
, command_context
, **options
):
442 self
.activate_virtualenv()
443 mozregression
= PypiBasedTool("mozregression")
444 mozregression
.run(**options
)
448 class NodeCommands(MachCommandBase
):
452 description
="Run the NodeJS interpreter used for building.",
454 @CommandArgument("args", nargs
=argparse
.REMAINDER
)
455 def node(self
, command_context
, args
):
456 from mozbuild
.nodeutil
import find_node_executable
458 # Avoid logging the command
459 self
.log_manager
.terminal_handler
.setLevel(logging
.CRITICAL
)
461 node_path
, _
= find_node_executable()
463 return self
.run_process(
465 pass_thru
=True, # Allow user to run Node interactively.
466 ensure_exit_code
=False, # Don't throw on non-zero exit code.
472 description
="Run the npm executable from the NodeJS used for building.",
474 @CommandArgument("args", nargs
=argparse
.REMAINDER
)
475 def npm(self
, command_context
, args
):
476 from mozbuild
.nodeutil
import find_npm_executable
478 # Avoid logging the command
479 self
.log_manager
.terminal_handler
.setLevel(logging
.CRITICAL
)
483 # Add node and npm from mozbuild to front of system path
485 # This isn't pretty, but npm currently executes itself with
486 # `#!/usr/bin/env node`, which means it just uses the node in the
487 # current PATH. As a result, stuff gets built wrong and installed
488 # in the wrong places and probably other badness too without this:
489 npm_path
, _
= find_npm_executable()
491 exit(-1, "could not find npm executable")
492 path
= os
.path
.abspath(os
.path
.dirname(npm_path
))
493 os
.environ
["PATH"] = "{}:{}".format(path
, os
.environ
["PATH"])
495 return self
.run_process(
496 [npm_path
, "--scripts-prepend-node-path=auto"] + args
,
497 pass_thru
=True, # Avoid eating npm output/error messages
498 ensure_exit_code
=False, # Don't throw on non-zero exit code.
502 def logspam_create_parser(subcommand
):
503 # Create the logspam command line parser.
504 # if logspam is not installed, or not up to date, it will
505 # first be installed.
506 loader
= PypiBasedTool("logspam", "mozilla-log-spam")
507 return loader
.create_parser(subcommand
)
510 from functools
import partial
514 class LogspamCommand(MachCommandBase
):
518 description
=("Warning categorizer for treeherder test runs."),
520 def logspam(self
, command_context
):
523 @SubCommand("logspam", "report", parser
=partial(logspam_create_parser
, "report"))
524 def report(self
, command_context
, **options
):
525 self
.activate_virtualenv()
526 logspam
= PypiBasedTool("logspam")
527 logspam
.run(command
="report", **options
)
529 @SubCommand("logspam", "bisect", parser
=partial(logspam_create_parser
, "bisect"))
530 def bisect(self
, command_context
, **options
):
531 self
.activate_virtualenv()
532 logspam
= PypiBasedTool("logspam")
533 logspam
.run(command
="bisect", **options
)
535 @SubCommand("logspam", "file", parser
=partial(logspam_create_parser
, "file"))
536 def create(self
, command_context
, **options
):
537 self
.activate_virtualenv()
538 logspam
= PypiBasedTool("logspam")
539 logspam
.run(command
="file", **options
)