Lib/shlex.py

   1 # -*- coding: iso-8859-1 -*-
   2 """A lexical analyzer class for simple shell-like syntaxes."""
   3
   4 # Module and documentation by Eric S. Raymond, 21 Dec 1998
   5 # Input stacking and error message cleanup added by ESR, March 2000
   6 # push_source() and pop_source() made explicit by ESR, January 2001.
   7 # Posix compliance, split(), string arguments, and
   8 # iterator interface by Gustavo Niemeyer, April 2003.
   9
  10 import os.path
  11 import sys
  12 from collections import deque
  13
  14 from io import StringIO
  15
  16 __all__ = ["shlex", "split"]
  17
  18 class shlex:
  19     "A lexical analyzer class for simple shell-like syntaxes."
  20     def __init__(self, instream=None, infile=None, posix=False):
  21         if isinstance(instream, str):
  22             instream = StringIO(instream)
  23         if instream is not None:
  24             self.instream = instream
  25             self.infile = infile
  26         else:
  27             self.instream = sys.stdin
  28             self.infile = None
  29         self.posix = posix
  30         if posix:
  31             self.eof = None
  32         else:
  33             self.eof = ''
  34         self.commenters = '#'
  35         self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
  36                           'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
  37         if self.posix:
  38             self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
  39                                'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
  40         self.whitespace = ' \t\r\n'
  41         self.whitespace_split = False
  42         self.quotes = '\'"'
  43         self.escape = '\\'
  44         self.escapedquotes = '"'
  45         self.state = ' '
  46         self.pushback = deque()
  47         self.lineno = 1
  48         self.debug = 0
  49         self.token = ''
  50         self.filestack = deque()
  51         self.source = None
  52         if self.debug:
  53             print('shlex: reading from %s, line %d' \
  54                   % (self.instream, self.lineno))
  55
  56     def push_token(self, tok):
  57         "Push a token onto the stack popped by the get_token method"
  58         if self.debug >= 1:
  59             print("shlex: pushing token " + repr(tok))
  60         self.pushback.appendleft(tok)
  61
  62     def push_source(self, newstream, newfile=None):
  63         "Push an input source onto the lexer's input source stack."
  64         if isinstance(newstream, str):
  65             newstream = StringIO(newstream)
  66         self.filestack.appendleft((self.infile, self.instream, self.lineno))
  67         self.infile = newfile
  68         self.instream = newstream
  69         self.lineno = 1
  70         if self.debug:
  71             if newfile is not None:
  72                 print('shlex: pushing to file %s' % (self.infile,))
  73             else:
  74                 print('shlex: pushing to stream %s' % (self.instream,))
  75
  76     def pop_source(self):
  77         "Pop the input source stack."
  78         self.instream.close()
  79         (self.infile, self.instream, self.lineno) = self.filestack.popleft()
  80         if self.debug:
  81             print('shlex: popping to %s, line %d' \
  82                   % (self.instream, self.lineno))
  83         self.state = ' '
  84
  85     def get_token(self):
  86         "Get a token from the input stream (or from stack if it's nonempty)"
  87         if self.pushback:
  88             tok = self.pushback.popleft()
  89             if self.debug >= 1:
  90                 print("shlex: popping token " + repr(tok))
  91             return tok
  92         # No pushback.  Get a token.
  93         raw = self.read_token()
  94         # Handle inclusions
  95         if self.source is not None:
  96             while raw == self.source:
  97                 spec = self.sourcehook(self.read_token())
  98                 if spec:
  99                     (newfile, newstream) = spec
 100                     self.push_source(newstream, newfile)
 101                 raw = self.get_token()
 102         # Maybe we got EOF instead?
 103         while raw == self.eof:
 104             if not self.filestack:
 105                 return self.eof
 106             else:
 107                 self.pop_source()
 108                 raw = self.get_token()
 109         # Neither inclusion nor EOF
 110         if self.debug >= 1:
 111             if raw != self.eof:
 112                 print("shlex: token=" + repr(raw))
 113             else:
 114                 print("shlex: token=EOF")
 115         return raw
 116
 117     def read_token(self):
 118         quoted = False
 119         escapedstate = ' '
 120         while True:
 121             nextchar = self.instream.read(1)
 122             if nextchar == '\n':
 123                 self.lineno = self.lineno + 1
 124             if self.debug >= 3:
 125                 print("shlex: in state", repr(self.state), \
 126                       "I see character:", repr(nextchar))
 127             if self.state is None:
 128                 self.token = ''        # past end of file
 129                 break
 130             elif self.state == ' ':
 131                 if not nextchar:
 132                     self.state = None  # end of file
 133                     break
 134                 elif nextchar in self.whitespace:
 135                     if self.debug >= 2:
 136                         print("shlex: I see whitespace in whitespace state")
 137                     if self.token or (self.posix and quoted):
 138                         break   # emit current token
 139                     else:
 140                         continue
 141                 elif nextchar in self.commenters:
 142                     self.instream.readline()
 143                     self.lineno = self.lineno + 1
 144                 elif self.posix and nextchar in self.escape:
 145                     escapedstate = 'a'
 146                     self.state = nextchar
 147                 elif nextchar in self.wordchars:
 148                     self.token = nextchar
 149                     self.state = 'a'
 150                 elif nextchar in self.quotes:
 151                     if not self.posix:
 152                         self.token = nextchar
 153                     self.state = nextchar
 154                 elif self.whitespace_split:
 155                     self.token = nextchar
 156                     self.state = 'a'
 157                 else:
 158                     self.token = nextchar
 159                     if self.token or (self.posix and quoted):
 160                         break   # emit current token
 161                     else:
 162                         continue
 163             elif self.state in self.quotes:
 164                 quoted = True
 165                 if not nextchar:      # end of file
 166                     if self.debug >= 2:
 167                         print("shlex: I see EOF in quotes state")
 168                     # XXX what error should be raised here?
 169                     raise ValueError("No closing quotation")
 170                 if nextchar == self.state:
 171                     if not self.posix:
 172                         self.token = self.token + nextchar
 173                         self.state = ' '
 174                         break
 175                     else:
 176                         self.state = 'a'
 177                 elif self.posix and nextchar in self.escape and \
 178                      self.state in self.escapedquotes:
 179                     escapedstate = self.state
 180                     self.state = nextchar
 181                 else:
 182                     self.token = self.token + nextchar
 183             elif self.state in self.escape:
 184                 if not nextchar:      # end of file
 185                     if self.debug >= 2:
 186                         print("shlex: I see EOF in escape state")
 187                     # XXX what error should be raised here?
 188                     raise ValueError("No escaped character")
 189                 # In posix shells, only the quote itself or the escape
 190                 # character may be escaped within quotes.
 191                 if escapedstate in self.quotes and \
 192                    nextchar != self.state and nextchar != escapedstate:
 193                     self.token = self.token + self.state
 194                 self.token = self.token + nextchar
 195                 self.state = escapedstate
 196             elif self.state == 'a':
 197                 if not nextchar:
 198                     self.state = None   # end of file
 199                     break
 200                 elif nextchar in self.whitespace:
 201                     if self.debug >= 2:
 202                         print("shlex: I see whitespace in word state")
 203                     self.state = ' '
 204                     if self.token or (self.posix and quoted):
 205                         break   # emit current token
 206                     else:
 207                         continue
 208                 elif nextchar in self.commenters:
 209                     self.instream.readline()
 210                     self.lineno = self.lineno + 1
 211                     if self.posix:
 212                         self.state = ' '
 213                         if self.token or (self.posix and quoted):
 214                             break   # emit current token
 215                         else:
 216                             continue
 217                 elif self.posix and nextchar in self.quotes:
 218                     self.state = nextchar
 219                 elif self.posix and nextchar in self.escape:
 220                     escapedstate = 'a'
 221                     self.state = nextchar
 222                 elif nextchar in self.wordchars or nextchar in self.quotes \
 223                     or self.whitespace_split:
 224                     self.token = self.token + nextchar
 225                 else:
 226                     self.pushback.appendleft(nextchar)
 227                     if self.debug >= 2:
 228                         print("shlex: I see punctuation in word state")
 229                     self.state = ' '
 230                     if self.token:
 231                         break   # emit current token
 232                     else:
 233                         continue
 234         result = self.token
 235         self.token = ''
 236         if self.posix and not quoted and result == '':
 237             result = None
 238         if self.debug > 1:
 239             if result:
 240                 print("shlex: raw token=" + repr(result))
 241             else:
 242                 print("shlex: raw token=EOF")
 243         return result
 244
 245     def sourcehook(self, newfile):
 246         "Hook called on a filename to be sourced."
 247         if newfile[0] == '"':
 248             newfile = newfile[1:-1]
 249         # This implements cpp-like semantics for relative-path inclusion.
 250         if isinstance(self.infile, str) and not os.path.isabs(newfile):
 251             newfile = os.path.join(os.path.dirname(self.infile), newfile)
 252         return (newfile, open(newfile, "r"))
 253
 254     def error_leader(self, infile=None, lineno=None):
 255         "Emit a C-compiler-like, Emacs-friendly error-message leader."
 256         if infile is None:
 257             infile = self.infile
 258         if lineno is None:
 259             lineno = self.lineno
 260         return "\"%s\", line %d: " % (infile, lineno)
 261
 262     def __iter__(self):
 263         return self
 264
 265     def __next__(self):
 266         token = self.get_token()
 267         if token == self.eof:
 268             raise StopIteration
 269         return token
 270
 271 def split(s, comments=False, posix=True):
 272     lex = shlex(s, posix=posix)
 273     lex.whitespace_split = True
 274     if not comments:
 275         lex.commenters = ''
 276     return list(lex)
 277
 278 if __name__ == '__main__':
 279     if len(sys.argv) == 1:
 280         lexer = shlex()
 281     else:
 282         file = sys.argv[1]
 283         lexer = shlex(open(file), file)
 284     while 1:
 285         tt = lexer.get_token()
 286         if tt:
 287             print("Token: " + repr(tt))
 288         else:
 289             break