Support end-of-line comments.
[pykickstart.git] / pykickstart / parser.py
blobc25fcef0760ba94c62bca72f5692ecb859cd7630
2 # parser.py: Kickstart file parser.
4 # Chris Lumens <clumens@redhat.com>
6 # Copyright 2005, 2006, 2007, 2008 Red Hat, Inc.
8 # This copyrighted material is made available to anyone wishing to use, modify,
9 # copy, or redistribute it subject to the terms and conditions of the GNU
10 # General Public License v.2. This program is distributed in the hope that it
11 # will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
12 # implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 # See the GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along with
16 # this program; if not, write to the Free Software Foundation, Inc., 51
17 # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
18 # trademarks that are incorporated in the source code or documentation are not
19 # subject to the GNU General Public License and may only be used or replicated
20 # with the express permission of Red Hat, Inc.
22 """
23 Main kickstart file processing module.
25 This module exports several important classes:
27 Script - Representation of a single %pre, %post, or %traceback script.
29 Packages - Representation of the %packages section.
31 KickstartParser - The kickstart file parser state machine.
32 """
34 import os
35 import shlex
36 import sys
37 import string
38 import tempfile
39 from copy import copy
40 from optparse import *
41 from sets import *
42 from urlgrabber import urlopen
43 import urlgrabber.grabber as grabber
45 from constants import *
46 from errors import *
47 from options import *
48 from version import *
50 from rhpl.translate import _
51 import rhpl.translate as translate
53 translate.textdomain("pykickstart")
55 STATE_END = 0
56 STATE_COMMANDS = 1
57 STATE_PACKAGES = 2
58 STATE_SCRIPT_HDR = 3
59 STATE_SCRIPT = 4
61 # FIXME: This is a hack until I have time to think about making the parser
62 # itself support multiple syntax versions. Yes, I know this means it will
63 # never be fixed.
64 ver = DEVEL
66 def _preprocessStateMachine (provideLineFn):
67 l = None
68 lineno = 0
70 # Now open an output kickstart file that we are going to write to one
71 # line at a time.
72 (outF, outName) = tempfile.mkstemp("-ks.cfg", "", "/tmp")
74 while True:
75 try:
76 l = provideLineFn()
77 except StopIteration:
78 break
80 # At the end of the file?
81 if l == "":
82 break
84 lineno += 1
85 url = None
87 ll = l.strip()
88 if string.find(ll, "%ksappend") == -1:
89 os.write(outF, l)
90 continue
92 # Try to pull down the remote file.
93 try:
94 ksurl = string.split(ll, ' ')[1]
95 except:
96 raise KickstartParseError, formatErrorMsg(lineno, msg=_("Illegal url for %%ksappend: %s") % ll)
98 try:
99 url = grabber.urlopen(ksurl)
100 except grabber.URLGrabError, e:
101 raise KickstartError, formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file: ") % e.strerror)
102 else:
103 # Sanity check result. Sometimes FTP doesn't catch a file
104 # is missing.
105 try:
106 if url.info()["content-length"] < 1:
107 raise KickstartError, formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file"))
108 except:
109 raise KickstartError, formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file"))
111 # If that worked, write the remote file to the output kickstart
112 # file in one burst. Then close everything up to get ready to
113 # read ahead in the input file. This allows multiple %ksappend
114 # lines to exist.
115 if url is not None:
116 os.write(outF, url.read())
117 url.close()
119 # All done - close the temp file and return its location.
120 os.close(outF)
121 return outName
123 def preprocessFromString (str):
124 """Preprocess the kickstart file, provided as the string str. This
125 method is currently only useful for handling %ksappend lines,
126 which need to be fetched before the real kickstart parser can be
127 run. Returns the location of the complete kickstart file.
129 i = iter(str.splitlines(True))
130 rc = _preprocessStateMachine (lambda: i.next())
131 return rc
133 def preprocessKickstart (file):
134 """Preprocess the kickstart file, given by the filename file. This
135 method is currently only useful for handling %ksappend lines,
136 which need to be fetched before the real kickstart parser can be
137 run. Returns the location of the complete kickstart file.
139 fh = urlopen(file)
140 rc = _preprocessStateMachine (lambda: fh.readline())
141 fh.close()
142 return rc
145 ### SCRIPT HANDLING
147 class Script:
148 """A class representing a single kickstart script. If functionality beyond
149 just a data representation is needed (for example, a run method in
150 anaconda), Script may be subclassed. Although a run method is not
151 provided, most of the attributes of Script have to do with running the
152 script. Instances of Script are held in a list by the Version object.
154 def __init__(self, script, interp = "/bin/sh", inChroot = False,
155 lineno = None, logfile = None, errorOnFail = False,
156 type = KS_SCRIPT_PRE):
157 """Create a new Script instance. Instance attributes:
159 errorOnFail -- If execution of the script fails, should anaconda
160 stop, display an error, and then reboot without
161 running any other scripts?
162 inChroot -- Does the script execute in anaconda's chroot
163 environment or not?
164 interp -- The program that should be used to interpret this
165 script.
166 lineno -- The line number this script starts on.
167 logfile -- Where all messages from the script should be logged.
168 script -- A string containing all the lines of the script.
169 type -- The type of the script, which can be KS_SCRIPT_* from
170 pykickstart.constants.
172 self.script = string.join(script, "")
173 self.interp = interp
174 self.inChroot = inChroot
175 self.lineno = lineno
176 self.logfile = logfile
177 self.errorOnFail = errorOnFail
178 self.type = type
180 def __str__(self):
181 """Return a string formatted for output to a kickstart file."""
182 if self.type == KS_SCRIPT_PRE:
183 retval = '\n%pre'
184 elif self.type == KS_SCRIPT_POST:
185 retval = '\n%post'
186 elif self.type == KS_SCRIPT_TRACEBACK:
187 retval = '\n%traceback'
189 if self.interp != "/bin/sh" and self.interp != "":
190 retval += " --interpreter=%s" % self.interp
191 if self.type == KS_SCRIPT_POST and not self.inChroot:
192 retval += " --nochroot"
193 if self.logfile != None:
194 retval += " --logfile %s" % self.logfile
195 if self.errorOnFail:
196 retval += " --erroronfail"
198 if self.script.endswith("\n"):
199 if ver >= F8:
200 return retval + "\n%s%%end\n" % self.script
201 else:
202 return retval + "\n%s\n" % self.script
203 else:
204 if ver >= F8:
205 return retval + "\n%s\n%%end\n" % self.script
206 else:
207 return retval + "\n%s\n" % self.script
211 ## PACKAGE HANDLING
213 class Group:
214 """A class representing a single group in the %packages section."""
215 def __init__(self, name="", include=GROUP_DEFAULT):
216 """Create a new Group instance. Instance attributes:
218 name -- The group's identifier
219 include -- The level of how much of the group should be included.
220 Values can be GROUP_* from pykickstart.constants.
222 self.name = name
223 self.include = include
225 def __str__(self):
226 """Return a string formatted for output to a kickstart file."""
227 if self.include == GROUP_REQUIRED:
228 return "@%s --nodefaults" % self.name
229 elif self.include == GROUP_ALL:
230 return "@%s --optional" % self.name
231 else:
232 return "@%s" % self.name
234 class Packages:
235 """A class representing the %packages section of the kickstart file."""
236 def __init__(self):
237 """Create a new Packages instance. Instance attributes:
239 addBase -- Should the Base group be installed even if it is
240 not specified?
241 default -- Should the default package set be selected?
242 excludedList -- A list of all the packages marked for exclusion in
243 the %packages section, without the leading minus
244 symbol.
245 excludeDocs -- Should documentation in each package be excluded?
246 groupList -- A list of Group objects representing all the groups
247 specified in the %packages section. Names will be
248 stripped of the leading @ symbol.
249 handleMissing -- If unknown packages are specified in the %packages
250 section, should it be ignored or not? Values can
251 be KS_MISSING_* from pykickstart.constants.
252 packageList -- A list of all the packages specified in the
253 %packages section.
254 instLangs -- A list of languages to install.
256 self.addBase = True
257 self.default = False
258 self.excludedList = []
259 self.excludeDocs = False
260 self.groupList = []
261 self.handleMissing = KS_MISSING_PROMPT
262 self.packageList = []
263 self.instLangs = None
265 def __str__(self):
266 """Return a string formatted for output to a kickstart file."""
267 pkgs = ""
269 if not self.default:
270 for grp in self.groupList:
271 pkgs += "%s\n" % grp.__str__()
273 for pkg in self.packageList:
274 pkgs += "%s\n" % pkg
276 for pkg in self.excludedList:
277 pkgs += "-%s\n" % pkg
279 if pkgs == "":
280 return ""
282 retval = "\n%packages"
284 if self.default:
285 retval += " --default"
286 if self.excludeDocs:
287 retval += " --excludedocs"
288 if not self.addBase:
289 retval += " --nobase"
290 if self.handleMissing == KS_MISSING_IGNORE:
291 retval += " --ignoremissing"
292 if self.instLangs:
293 retval += " --instLangs=%s" % self.instLangs
295 if ver >= F8:
296 return retval + "\n" + pkgs + "\n%end\n"
297 else:
298 return retval + "\n" + pkgs + "\n"
300 def _processGroup (self, line):
301 op = OptionParser()
302 op.add_option("--nodefaults", action="store_true", default=False)
303 op.add_option("--optional", action="store_true", default=False)
305 (opts, extra) = op.parse_args(args=line.split())
307 if opts.nodefaults and opts.optional:
308 raise KickstartValueError, _("Group cannot specify both --nodefaults and --optional")
310 # If the group name has spaces in it, we have to put it back together
311 # now.
312 grp = " ".join(extra)
314 if opts.nodefaults:
315 self.groupList.append(Group(name=grp, include=GROUP_REQUIRED))
316 elif opts.optional:
317 self.groupList.append(Group(name=grp, include=GROUP_ALL))
318 else:
319 self.groupList.append(Group(name=grp, include=GROUP_DEFAULT))
321 def add (self, pkgList):
322 """Given a list of lines from the input file, strip off any leading
323 symbols and add the result to the appropriate list.
325 existingExcludedSet = Set(self.excludedList)
326 existingPackageSet = Set(self.packageList)
327 newExcludedSet = Set()
328 newPackageSet = Set()
330 for pkg in pkgList:
331 stripped = pkg.strip()
333 if stripped[0] == "@":
334 self._processGroup(stripped[1:])
335 elif stripped[0] == "-":
336 newExcludedSet.add(stripped[1:])
337 else:
338 newPackageSet.add(stripped)
340 existingPackageSet = (existingPackageSet - newExcludedSet) | newPackageSet
341 existingExcludedSet = (existingExcludedSet - existingPackageSet) | newExcludedSet
343 self.packageList = list(existingPackageSet)
344 self.excludedList = list(existingExcludedSet)
348 ### PARSER
350 class KickstartParser:
351 """The kickstart file parser class as represented by a basic state
352 machine. To create a specialized parser, make a subclass and override
353 any of the methods you care about. Methods that don't need to do
354 anything may just pass. However, _stateMachine should never be
355 overridden.
357 def __init__ (self, handler, followIncludes=True, errorsAreFatal=True,
358 missingIncludeIsFatal=True):
359 """Create a new KickstartParser instance. Instance attributes:
361 errorsAreFatal -- Should errors cause processing to halt, or
362 just print a message to the screen? This
363 is most useful for writing syntax checkers
364 that may want to continue after an error is
365 encountered.
366 followIncludes -- If %include is seen, should the included
367 file be checked as well or skipped?
368 handler -- An instance of a BaseHandler subclass. If
369 None, the input file will still be parsed
370 but no data will be saved and no commands
371 will be executed.
372 missingIncludeIsFatal -- Should missing include files be fatal, even
373 if errorsAreFatal is False?
375 self.errorsAreFatal = errorsAreFatal
376 self.followIncludes = followIncludes
377 self.handler = handler
378 self.currentdir = {}
379 self.missingIncludeIsFatal = missingIncludeIsFatal
380 self._reset()
382 self._line = ""
384 self.version = self.handler.version
386 global ver
387 ver = self.version
389 def _reset(self):
390 """Reset the internal variables of the state machine for a new kickstart file."""
391 self._state = STATE_COMMANDS
392 self._script = None
393 self._includeDepth = 0
395 def addScript (self):
396 """Create a new Script instance and add it to the Version object. This
397 is called when the end of a script section is seen and may be
398 overridden in a subclass if necessary.
400 if string.join(self._script["body"]).strip() == "":
401 return
403 s = Script (self._script["body"], interp=self._script["interp"],
404 inChroot=self._script["chroot"],
405 lineno=self._script["lineno"],
406 logfile=self._script["log"],
407 errorOnFail=self._script["errorOnFail"],
408 type=self._script["type"])
410 if self.handler:
411 self.handler.scripts.append(s)
413 def addPackages (self, line):
414 """Add the single package, exclude, or group into the Version's
415 Packages instance. This method may be overridden in a subclass
416 if necessary.
418 if self.handler:
419 self.handler.packages.add([line])
421 def handleCommand (self, lineno, args):
422 """Given the list of command and arguments, call the Version's
423 dispatcher method to handle the command. This method may be
424 overridden in a subclass if necessary.
426 if self.handler:
427 self.handler.currentCmd = args[0]
428 self.handler.currentLine = self._line
429 self.handler.dispatcher(args, lineno)
431 def handlePackageHdr (self, lineno, args):
432 """Process the arguments to the %packages header and set attributes
433 on the Version's Packages instance appropriate. This method may be
434 overridden in a subclass if necessary.
436 op = KSOptionParser(lineno=lineno, version=self.version)
437 op.add_option("--excludedocs", dest="excludedocs", action="store_true",
438 default=False)
439 op.add_option("--ignoremissing", dest="ignoremissing",
440 action="store_true", default=False)
441 op.add_option("--nobase", dest="nobase", action="store_true",
442 default=False)
443 op.add_option("--ignoredeps", dest="resolveDeps", action="store_false",
444 deprecated=FC4, removed=F9)
445 op.add_option("--resolvedeps", dest="resolveDeps", action="store_true",
446 deprecated=FC4, removed=F9)
447 op.add_option("--default", dest="defaultPackages", action="store_true",
448 default=False, introduced=F7)
449 op.add_option("--instLangs", dest="instLangs", type="string",
450 default="", introduced=F9)
452 (opts, extra) = op.parse_args(args=args[1:])
454 self.handler.packages.excludeDocs = opts.excludedocs
455 self.handler.packages.addBase = not opts.nobase
456 if opts.ignoremissing:
457 self.handler.packages.handleMissing = KS_MISSING_IGNORE
458 else:
459 self.handler.packages.handleMissing = KS_MISSING_PROMPT
461 if opts.defaultPackages:
462 self.handler.packages.default = True
464 if opts.instLangs:
465 self.handler.packages.instLange = opts.instLangs
467 def handleScriptHdr (self, lineno, args):
468 """Process the arguments to a %pre/%post/%traceback header for later
469 setting on a Script instance once the end of the script is found.
470 This method may be overridden in a subclass if necessary.
472 op = KSOptionParser(lineno=lineno, version=self.version)
473 op.add_option("--erroronfail", dest="errorOnFail", action="store_true",
474 default=False)
475 op.add_option("--interpreter", dest="interpreter", default="/bin/sh")
476 op.add_option("--log", "--logfile", dest="log")
478 if args[0] == "%pre" or args[0] == "%traceback":
479 self._script["chroot"] = False
480 elif args[0] == "%post":
481 self._script["chroot"] = True
482 op.add_option("--nochroot", dest="nochroot", action="store_true",
483 default=False)
485 (opts, extra) = op.parse_args(args=args[1:])
487 self._script["interp"] = opts.interpreter
488 self._script["lineno"] = lineno
489 self._script["log"] = opts.log
490 self._script["errorOnFail"] = opts.errorOnFail
491 if hasattr(opts, "nochroot"):
492 self._script["chroot"] = not opts.nochroot
494 def _stateMachine (self, provideLineFn):
495 # For error reporting.
496 lineno = 0
497 needLine = True
499 while True:
500 if needLine:
501 try:
502 self._line = provideLineFn()
503 except StopIteration:
504 break
506 lineno += 1
507 needLine = False
509 # At the end of an included file
510 if self._line == "" and self._includeDepth > 0:
511 break
513 # Don't eliminate whitespace or comments from scripts.
514 if self._line.isspace() or (self._line != "" and self._line.lstrip()[0] == '#'):
515 # Save the platform for s-c-kickstart, though.
516 if self._line[:10] == "#platform=" and self._state == STATE_COMMANDS:
517 self.handler.platform = self._line[11:]
519 if self._state == STATE_SCRIPT:
520 self._script["body"].append(self._line)
522 needLine = True
523 continue
525 # We only want to split the line if we're outside of a script,
526 # as inside the script might involve some pretty weird quoting
527 # that shlex doesn't understand.
528 if self._state == STATE_SCRIPT:
529 # Have we found a state transition? If so, we still want
530 # to split. Otherwise, args won't be set but we'll fall through
531 # all the way to the last case.
532 if self._line != "" and string.split(self._line.lstrip())[0] in \
533 ["%end", "%post", "%pre", "%traceback", "%include", "%packages", "%ksappend"]:
534 args = shlex.split(self._line)
535 else:
536 args = None
537 else:
538 # Remove any end-of-line comments.
539 (h, s, t) = self._line.partition("#")
540 self._line = h.rstrip()
541 args = shlex.split(self._line)
543 if args and args[0] == "%include":
544 # This case comes up primarily in ksvalidator.
545 if not self.followIncludes:
546 needLine = True
547 continue
549 if not args[1]:
550 raise KickstartParseError, formatErrorMsg(lineno)
551 else:
552 self._includeDepth += 1
554 try:
555 self.readKickstart (args[1], reset=False)
556 except IOError:
557 # Handle the include file being provided over the
558 # network in a %pre script. This case comes up in the
559 # early parsing in anaconda.
560 if self.missingIncludeIsFatal:
561 raise
563 self._includeDepth -= 1
564 needLine = True
565 continue
567 if self._state == STATE_COMMANDS:
568 if not args and self._includeDepth == 0:
569 self._state = STATE_END
570 elif args[0] == "%ksappend":
571 needLine = True
572 elif args[0] in ["%pre", "%post", "%traceback"]:
573 self._state = STATE_SCRIPT_HDR
574 elif args[0] == "%packages":
575 self._state = STATE_PACKAGES
576 elif args[0][0] == '%':
577 # This error is too difficult to continue from, without
578 # lots of resync code. So just print this one and quit.
579 raise KickstartParseError, formatErrorMsg(lineno)
580 else:
581 needLine = True
583 if self.errorsAreFatal:
584 self.handleCommand(lineno, args)
585 else:
586 try:
587 self.handleCommand(lineno, args)
588 except Exception, msg:
589 print msg
591 elif self._state == STATE_PACKAGES:
592 if not args and self._includeDepth == 0:
593 if self.version >= F8 :
594 warnings.warn(_("%s does not end with %%end. This syntax has been deprecated. It may be removed from future releases, which will result in a fatal error from kickstart. Please modify your kickstart file to use this updated syntax.") % "%packages", DeprecationWarning)
596 self._state = STATE_END
597 elif args[0] == "%end":
598 self._state = STATE_COMMANDS
599 needLine = True
600 elif args[0] == "%ksappend":
601 needLine = True
602 elif args[0] in ["%pre", "%post", "%traceback"]:
603 self._state = STATE_SCRIPT_HDR
604 elif args[0] == "%packages":
605 needLine = True
607 if self.errorsAreFatal:
608 self.handlePackageHdr (lineno, args)
609 else:
610 try:
611 self.handlePackageHdr (lineno, args)
612 except Exception, msg:
613 print msg
614 elif args[0][0] == '%':
615 # This error is too difficult to continue from, without
616 # lots of resync code. So just print this one and quit.
617 raise KickstartParseError, formatErrorMsg(lineno)
618 else:
619 needLine = True
620 self.addPackages (string.rstrip(self._line))
622 elif self._state == STATE_SCRIPT_HDR:
623 needLine = True
624 self._script = {"body": [], "interp": "/bin/sh", "log": None,
625 "errorOnFail": False, lineno: None}
627 if not args and self._includeDepth == 0:
628 self._state = STATE_END
629 elif args[0] == "%pre":
630 self._state = STATE_SCRIPT
631 self._script["type"] = KS_SCRIPT_PRE
632 elif args[0] == "%post":
633 self._state = STATE_SCRIPT
634 self._script["type"] = KS_SCRIPT_POST
635 elif args[0] == "%traceback":
636 self._state = STATE_SCRIPT
637 self._script["type"] = KS_SCRIPT_TRACEBACK
638 elif args[0][0] == '%':
639 # This error is too difficult to continue from, without
640 # lots of resync code. So just print this one and quit.
641 raise KickstartParseError, formatErrorMsg(lineno)
643 if self.errorsAreFatal:
644 self.handleScriptHdr (lineno, args)
645 else:
646 try:
647 self.handleScriptHdr (lineno, args)
648 except Exception, msg:
649 print msg
651 elif self._state == STATE_SCRIPT:
652 if self._line in ["%end", ""] and self._includeDepth == 0:
653 if self._line == "" and self.version >= F8:
654 warnings.warn(_("%s does not end with %%end. This syntax has been deprecated. It may be removed from future releases, which will result in a fatal error from kickstart. Please modify your kickstart file to use this updated syntax.") % _("Script"), DeprecationWarning)
656 # If we're at the end of the kickstart file, add the script.
657 self.addScript()
658 self._state = STATE_END
659 elif args and args[0] in ["%end", "%pre", "%post", "%traceback", "%packages", "%ksappend"]:
660 # Otherwise we're now at the start of the next section.
661 # Figure out what kind of a script we just finished
662 # reading, add it to the list, and switch to the initial
663 # state.
664 self.addScript()
665 self._state = STATE_COMMANDS
667 if args[0] == "%end":
668 needLine = True
669 else:
670 # Otherwise just add to the current script body.
671 self._script["body"].append(self._line)
672 needLine = True
674 elif self._state == STATE_END:
675 break
677 def readKickstartFromString (self, str, reset=True):
678 """Process a kickstart file, provided as the string str."""
679 if reset:
680 self._reset()
682 i = iter(str.splitlines(True))
683 self._stateMachine (lambda: i.next())
685 def readKickstart(self, file, reset=True):
686 """Process a kickstart file, given by the filename file."""
687 if reset:
688 self._reset()
690 # an %include might not specify a full path. if we don't try to figure
691 # out what the path should have been, then we're unable to find it
692 # requiring full path specification, though, sucks. so let's make
693 # the reading "smart" by keeping track of what the path is at each
694 # include depth.
695 if not os.path.exists(file):
696 if self.currentdir.has_key(self._includeDepth - 1):
697 if os.path.exists(os.path.join(self.currentdir[self._includeDepth - 1], file)):
698 file = os.path.join(self.currentdir[self._includeDepth - 1], file)
700 cd = os.path.dirname(file)
701 if not cd.startswith("/"):
702 cd = os.path.abspath(cd)
703 self.currentdir[self._includeDepth] = cd
705 fh = urlopen(file)
706 self._stateMachine (lambda: fh.readline())
707 fh.close()