python/mozbuild/mozbuild/shellutil.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   3 # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import re
   6
   7
   8 def _tokens2re(**tokens):
   9     # Create a pattern for non-escaped tokens, in the form:
  10     #   (?<!\\)(?:a|b|c...)
  11     # This is meant to match patterns a, b, or c, or ... if they are not
  12     # preceded by a backslash.
  13     # where a, b, c... are in the form
  14     #   (?P<name>pattern)
  15     # which matches the pattern and captures it in a named match group.
  16     # The group names and patterns are given as arguments.
  17     all_tokens = "|".join(
  18         "(?P<%s>%s)" % (name, value) for name, value in tokens.items()
  19     )
  20     nonescaped = r"(?<!\\)(?:%s)" % all_tokens
  21
  22     # The final pattern matches either the above pattern, or an escaped
  23     # backslash, captured in the "escape" match group.
  24     return re.compile("(?:%s|%s)" % (nonescaped, r"(?P<escape>\\\\)"))
  25
  26
  27 UNQUOTED_TOKENS_RE = _tokens2re(
  28     whitespace=r"[\t\r\n ]+",
  29     quote=r'[\'"]',
  30     comment="#",
  31     special=r"[<>&|`(){}$;\*\?]",
  32     backslashed=r"\\[^\\]",
  33 )
  34
  35 DOUBLY_QUOTED_TOKENS_RE = _tokens2re(
  36     quote='"',
  37     backslashedquote=r'\\"',
  38     special=r"\$",
  39     backslashed=r'\\[^\\"]',
  40 )
  41
  42 ESCAPED_NEWLINES_RE = re.compile(r"\\\n")
  43
  44 # This regexp contains the same characters as all those listed in
  45 # UNQUOTED_TOKENS_RE. Please keep in sync.
  46 SHELL_QUOTE_RE = re.compile(r"[\\\t\r\n \'\"#<>&|`(){}$;\*\?]")
  47
  48
  49 class MetaCharacterException(Exception):
  50     def __init__(self, char):
  51         self.char = char
  52
  53
  54 class _ClineSplitter(object):
  55     """
  56     Parses a given command line string and creates a list of command
  57     and arguments, with wildcard expansion.
  58     """
  59
  60     def __init__(self, cline):
  61         self.arg = None
  62         self.cline = cline
  63         self.result = []
  64         self._parse_unquoted()
  65
  66     def _push(self, str):
  67         """
  68         Push the given string as part of the current argument
  69         """
  70         if self.arg is None:
  71             self.arg = ""
  72         self.arg += str
  73
  74     def _next(self):
  75         """
  76         Finalize current argument, effectively adding it to the list.
  77         """
  78         if self.arg is None:
  79             return
  80         self.result.append(self.arg)
  81         self.arg = None
  82
  83     def _parse_unquoted(self):
  84         """
  85         Parse command line remainder in the context of an unquoted string.
  86         """
  87         while self.cline:
  88             # Find the next token
  89             m = UNQUOTED_TOKENS_RE.search(self.cline)
  90             # If we find none, the remainder of the string can be pushed to
  91             # the current argument and the argument finalized
  92             if not m:
  93                 self._push(self.cline)
  94                 break
  95             # The beginning of the string, up to the found token, is part of
  96             # the current argument
  97             if m.start():
  98                 self._push(self.cline[: m.start()])
  99             self.cline = self.cline[m.end() :]
 100
 101             match = {name: value for name, value in m.groupdict().items() if value}
 102             if "quote" in match:
 103                 # " or ' start a quoted string
 104                 if match["quote"] == '"':
 105                     self._parse_doubly_quoted()
 106                 else:
 107                     self._parse_quoted()
 108             elif "comment" in match:
 109                 # Comments are ignored. The current argument can be finalized,
 110                 # and parsing stopped.
 111                 break
 112             elif "special" in match:
 113                 # Unquoted, non-escaped special characters need to be sent to a
 114                 # shell.
 115                 raise MetaCharacterException(match["special"])
 116             elif "whitespace" in match:
 117                 # Whitespaces terminate current argument.
 118                 self._next()
 119             elif "escape" in match:
 120                 # Escaped backslashes turn into a single backslash
 121                 self._push("\\")
 122             elif "backslashed" in match:
 123                 # Backslashed characters are unbackslashed
 124                 # e.g. echo \a -> a
 125                 self._push(match["backslashed"][1])
 126             else:
 127                 raise Exception("Shouldn't reach here")
 128         if self.arg:
 129             self._next()
 130
 131     def _parse_quoted(self):
 132         # Single quoted strings are preserved, except for the final quote
 133         index = self.cline.find("'")
 134         if index == -1:
 135             raise Exception("Unterminated quoted string in command")
 136         self._push(self.cline[:index])
 137         self.cline = self.cline[index + 1 :]
 138
 139     def _parse_doubly_quoted(self):
 140         if not self.cline:
 141             raise Exception("Unterminated quoted string in command")
 142         while self.cline:
 143             m = DOUBLY_QUOTED_TOKENS_RE.search(self.cline)
 144             if not m:
 145                 raise Exception("Unterminated quoted string in command")
 146             self._push(self.cline[: m.start()])
 147             self.cline = self.cline[m.end() :]
 148             match = {name: value for name, value in m.groupdict().items() if value}
 149             if "quote" in match:
 150                 # a double quote ends the quoted string, so go back to
 151                 # unquoted parsing
 152                 return
 153             elif "special" in match:
 154                 # Unquoted, non-escaped special characters in a doubly quoted
 155                 # string still have a special meaning and need to be sent to a
 156                 # shell.
 157                 raise MetaCharacterException(match["special"])
 158             elif "escape" in match:
 159                 # Escaped backslashes turn into a single backslash
 160                 self._push("\\")
 161             elif "backslashedquote" in match:
 162                 # Backslashed double quotes are un-backslashed
 163                 self._push('"')
 164             elif "backslashed" in match:
 165                 # Backslashed characters are kept backslashed
 166                 self._push(match["backslashed"])
 167
 168
 169 def split(cline):
 170     """
 171     Split the given command line string.
 172     """
 173     s = ESCAPED_NEWLINES_RE.sub("", cline)
 174     return _ClineSplitter(s).result
 175
 176
 177 def _quote(s):
 178     """Given a string, returns a version that can be used literally on a shell
 179     command line, enclosing it with single quotes if necessary.
 180
 181     As a special case, if given an int, returns a string containing the int,
 182     not enclosed in quotes.
 183     """
 184     if type(s) == int:
 185         return f"{s}"
 186
 187     # Empty strings need to be quoted to have any significance
 188     if s and not SHELL_QUOTE_RE.search(s) and s[0] != "~":
 189         return s
 190
 191     # Single quoted strings can contain any characters unescaped except the
 192     # single quote itself, which can't even be escaped, so the string needs to
 193     # be closed, an escaped single quote added, and reopened.
 194     t = type(s)
 195     return t("'%s'") % s.replace(t("'"), t("'\\''"))
 196
 197
 198 def quote(*strings):
 199     """Given one or more strings, returns a quoted string that can be used
 200     literally on a shell command line.
 201
 202         >>> quote('a', 'b')
 203         "a b"
 204         >>> quote('a b', 'c')
 205         "'a b' c"
 206     """
 207     return " ".join(_quote(s) for s in strings)
 208
 209
 210 __all__ = ["MetaCharacterException", "split", "quote"]