It's url.hdr now, not url.info.
[pykickstart.git] / pykickstart / parser.py
blob389cf70fe8e42141c429a68d72c672f7071074bc
2 # parser.py: Kickstart file parser.
4 # Chris Lumens <clumens@redhat.com>
6 # Copyright 2005, 2006, 2007, 2008, 2011 Red Hat, Inc.
8 # This copyrighted material is made available to anyone wishing to use, modify,
9 # copy, or redistribute it subject to the terms and conditions of the GNU
10 # General Public License v.2. This program is distributed in the hope that it
11 # will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
12 # implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 # See the GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along with
16 # this program; if not, write to the Free Software Foundation, Inc., 51
17 # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
18 # trademarks that are incorporated in the source code or documentation are not
19 # subject to the GNU General Public License and may only be used or replicated
20 # with the express permission of Red Hat, Inc.
22 """
23 Main kickstart file processing module.
25 This module exports several important classes:
27 Script - Representation of a single %pre, %post, or %traceback script.
29 Packages - Representation of the %packages section.
31 KickstartParser - The kickstart file parser state machine.
32 """
34 from collections import Iterator
35 import os
36 import shlex
37 import sys
38 import tempfile
39 from copy import copy
40 from optparse import *
41 from urlgrabber import urlread
42 import urlgrabber.grabber as grabber
44 import constants
45 from errors import KickstartError, KickstartParseError, KickstartValueError, formatErrorMsg
46 from ko import KickstartObject
47 from sections import *
48 import version
50 import gettext
51 _ = lambda x: gettext.ldgettext("pykickstart", x)
53 STATE_END = "end"
54 STATE_COMMANDS = "commands"
56 ver = version.DEVEL
58 def _preprocessStateMachine (lineIter):
59 l = None
60 lineno = 0
62 # Now open an output kickstart file that we are going to write to one
63 # line at a time.
64 (outF, outName) = tempfile.mkstemp("-ks.cfg", "", "/tmp")
66 while True:
67 try:
68 l = lineIter.next()
69 except StopIteration:
70 break
72 # At the end of the file?
73 if l == "":
74 break
76 lineno += 1
77 url = None
79 ll = l.strip()
80 if not ll.startswith("%ksappend"):
81 os.write(outF, l)
82 continue
84 # Try to pull down the remote file.
85 try:
86 ksurl = ll.split(' ')[1]
87 except:
88 raise KickstartParseError, formatErrorMsg(lineno, msg=_("Illegal url for %%ksappend: %s") % ll)
90 try:
91 url = grabber.urlopen(ksurl)
92 except grabber.URLGrabError, e:
93 raise KickstartError, formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file: %s") % e.strerror)
94 else:
95 # Sanity check result. Sometimes FTP doesn't catch a file
96 # is missing.
97 try:
98 if url.hdr["content-length"] < 1:
99 raise KickstartError, formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file"))
100 except:
101 raise KickstartError, formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file"))
103 # If that worked, write the remote file to the output kickstart
104 # file in one burst. Then close everything up to get ready to
105 # read ahead in the input file. This allows multiple %ksappend
106 # lines to exist.
107 if url is not None:
108 os.write(outF, url.read())
109 url.close()
111 # All done - close the temp file and return its location.
112 os.close(outF)
113 return outName
115 def preprocessFromString (s):
116 """Preprocess the kickstart file, provided as the string str. This
117 method is currently only useful for handling %ksappend lines,
118 which need to be fetched before the real kickstart parser can be
119 run. Returns the location of the complete kickstart file.
121 i = iter(s.splitlines(True) + [""])
122 rc = _preprocessStateMachine (i.next)
123 return rc
125 def preprocessKickstart (f):
126 """Preprocess the kickstart file, given by the filename file. This
127 method is currently only useful for handling %ksappend lines,
128 which need to be fetched before the real kickstart parser can be
129 run. Returns the location of the complete kickstart file.
131 try:
132 fh = urlopen(f)
133 except grabber.URLGrabError, e:
134 raise KickstartError, formatErrorMsg(0, msg=_("Unable to open input kickstart file: %s") % e.strerror)
136 rc = _preprocessStateMachine (iter(fh.readlines()))
137 fh.close()
138 return rc
140 class PutBackIterator(Iterator):
141 def __init__(self, iterable):
142 self._iterable = iter(iterable)
143 self._buf = None
145 def __iter__(self):
146 return self
148 def put(self, s):
149 self._buf = s
151 def next(self):
152 if self._buf:
153 retval = self._buf
154 self._buf = None
155 return retval
156 else:
157 return self._iterable.next()
160 ### SCRIPT HANDLING
162 class Script(KickstartObject):
163 """A class representing a single kickstart script. If functionality beyond
164 just a data representation is needed (for example, a run method in
165 anaconda), Script may be subclassed. Although a run method is not
166 provided, most of the attributes of Script have to do with running the
167 script. Instances of Script are held in a list by the Version object.
169 def __init__(self, script, *args , **kwargs):
170 """Create a new Script instance. Instance attributes:
172 errorOnFail -- If execution of the script fails, should anaconda
173 stop, display an error, and then reboot without
174 running any other scripts?
175 inChroot -- Does the script execute in anaconda's chroot
176 environment or not?
177 interp -- The program that should be used to interpret this
178 script.
179 lineno -- The line number this script starts on.
180 logfile -- Where all messages from the script should be logged.
181 script -- A string containing all the lines of the script.
182 type -- The type of the script, which can be KS_SCRIPT_* from
183 pykickstart.constants.
185 KickstartObject.__init__(self, *args, **kwargs)
186 self.script = "".join(script)
188 self.interp = kwargs.get("interp", "/bin/sh")
189 self.inChroot = kwargs.get("inChroot", False)
190 self.lineno = kwargs.get("lineno", None)
191 self.logfile = kwargs.get("logfile", None)
192 self.errorOnFail = kwargs.get("errorOnFail", False)
193 self.type = kwargs.get("type", constants.KS_SCRIPT_PRE)
195 def __str__(self):
196 """Return a string formatted for output to a kickstart file."""
197 retval = ""
199 if self.type == constants.KS_SCRIPT_PRE:
200 retval += '\n%pre'
201 elif self.type == constants.KS_SCRIPT_POST:
202 retval += '\n%post'
203 elif self.type == constants.KS_SCRIPT_TRACEBACK:
204 retval += '\n%traceback'
206 if self.interp != "/bin/sh" and self.interp != "":
207 retval += " --interpreter=%s" % self.interp
208 if self.type == constants.KS_SCRIPT_POST and not self.inChroot:
209 retval += " --nochroot"
210 if self.logfile != None:
211 retval += " --logfile %s" % self.logfile
212 if self.errorOnFail:
213 retval += " --erroronfail"
215 if self.script.endswith("\n"):
216 if ver >= version.F8:
217 return retval + "\n%s%%end\n" % self.script
218 else:
219 return retval + "\n%s\n" % self.script
220 else:
221 if ver >= version.F8:
222 return retval + "\n%s\n%%end\n" % self.script
223 else:
224 return retval + "\n%s\n" % self.script
228 ## PACKAGE HANDLING
230 class Group:
231 """A class representing a single group in the %packages section."""
232 def __init__(self, name="", include=constants.GROUP_DEFAULT):
233 """Create a new Group instance. Instance attributes:
235 name -- The group's identifier
236 include -- The level of how much of the group should be included.
237 Values can be GROUP_* from pykickstart.constants.
239 self.name = name
240 self.include = include
242 def __str__(self):
243 """Return a string formatted for output to a kickstart file."""
244 if self.include == constants.GROUP_REQUIRED:
245 return "@%s --nodefaults" % self.name
246 elif self.include == constants.GROUP_ALL:
247 return "@%s --optional" % self.name
248 else:
249 return "@%s" % self.name
251 def __cmp__(self, other):
252 if self.name < other.name:
253 return -1
254 elif self.name > other.name:
255 return 1
256 return 0
258 class Packages(KickstartObject):
259 """A class representing the %packages section of the kickstart file."""
260 def __init__(self, *args, **kwargs):
261 """Create a new Packages instance. Instance attributes:
263 addBase -- Should the Base group be installed even if it is
264 not specified?
265 default -- Should the default package set be selected?
266 excludedList -- A list of all the packages marked for exclusion in
267 the %packages section, without the leading minus
268 symbol.
269 excludeDocs -- Should documentation in each package be excluded?
270 groupList -- A list of Group objects representing all the groups
271 specified in the %packages section. Names will be
272 stripped of the leading @ symbol.
273 excludedGroupList -- A list of Group objects representing all the
274 groups specified for removal in the %packages
275 section. Names will be stripped of the leading
276 -@ symbols.
277 handleMissing -- If unknown packages are specified in the %packages
278 section, should it be ignored or not? Values can
279 be KS_MISSING_* from pykickstart.constants.
280 packageList -- A list of all the packages specified in the
281 %packages section.
282 instLangs -- A list of languages to install.
284 KickstartObject.__init__(self, *args, **kwargs)
286 self.addBase = True
287 self.default = False
288 self.excludedList = []
289 self.excludedGroupList = []
290 self.excludeDocs = False
291 self.groupList = []
292 self.handleMissing = constants.KS_MISSING_PROMPT
293 self.packageList = []
294 self.instLangs = None
296 def __str__(self):
297 """Return a string formatted for output to a kickstart file."""
298 pkgs = ""
300 if not self.default:
301 grps = self.groupList
302 grps.sort()
303 for grp in grps:
304 pkgs += "%s\n" % grp.__str__()
306 p = self.packageList
307 p.sort()
308 for pkg in p:
309 pkgs += "%s\n" % pkg
311 grps = self.excludedGroupList
312 grps.sort()
313 for grp in grps:
314 pkgs += "-%s\n" % grp.__str__()
316 p = self.excludedList
317 p.sort()
318 for pkg in p:
319 pkgs += "-%s\n" % pkg
321 if pkgs == "":
322 return ""
324 retval = "\n%packages"
326 if self.default:
327 retval += " --default"
328 if self.excludeDocs:
329 retval += " --excludedocs"
330 if not self.addBase:
331 retval += " --nobase"
332 if self.handleMissing == constants.KS_MISSING_IGNORE:
333 retval += " --ignoremissing"
334 if self.instLangs:
335 retval += " --instLangs=%s" % self.instLangs
337 if ver >= version.F8:
338 return retval + "\n" + pkgs + "\n%end\n"
339 else:
340 return retval + "\n" + pkgs + "\n"
342 def _processGroup (self, line):
343 op = OptionParser()
344 op.add_option("--nodefaults", action="store_true", default=False)
345 op.add_option("--optional", action="store_true", default=False)
347 (opts, extra) = op.parse_args(args=line.split())
349 if opts.nodefaults and opts.optional:
350 raise KickstartValueError, _("Group cannot specify both --nodefaults and --optional")
352 # If the group name has spaces in it, we have to put it back together
353 # now.
354 grp = " ".join(extra)
356 if opts.nodefaults:
357 self.groupList.append(Group(name=grp, include=constants.GROUP_REQUIRED))
358 elif opts.optional:
359 self.groupList.append(Group(name=grp, include=constants.GROUP_ALL))
360 else:
361 self.groupList.append(Group(name=grp, include=constants.GROUP_DEFAULT))
363 def add (self, pkgList):
364 """Given a list of lines from the input file, strip off any leading
365 symbols and add the result to the appropriate list.
367 existingExcludedSet = set(self.excludedList)
368 existingPackageSet = set(self.packageList)
369 newExcludedSet = set()
370 newPackageSet = set()
372 excludedGroupList = []
374 for pkg in pkgList:
375 stripped = pkg.strip()
377 if stripped[0] == "@":
378 self._processGroup(stripped[1:])
379 elif stripped[0] == "-":
380 if stripped[1] == "@":
381 excludedGroupList.append(Group(name=stripped[2:]))
382 else:
383 newExcludedSet.add(stripped[1:])
384 else:
385 newPackageSet.add(stripped)
387 # Groups have to be excluded in two different ways (note: can't use
388 # sets here because we have to store objects):
389 excludedGroupNames = map(lambda g: g.name, excludedGroupList)
391 # First, an excluded group may be cancelling out a previously given
392 # one. This is often the case when using %include. So there we should
393 # just remove the group from the list.
394 self.groupList = filter(lambda g: g.name not in excludedGroupNames, self.groupList)
396 # Second, the package list could have included globs which are not
397 # processed by pykickstart. In that case we need to preserve a list of
398 # excluded groups so whatever tool doing package/group installation can
399 # take appropriate action.
400 self.excludedGroupList.extend(excludedGroupList)
402 existingPackageSet = (existingPackageSet - newExcludedSet) | newPackageSet
403 existingExcludedSet = (existingExcludedSet - existingPackageSet) | newExcludedSet
405 self.packageList = list(existingPackageSet)
406 self.excludedList = list(existingExcludedSet)
410 ### PARSER
412 class KickstartParser:
413 """The kickstart file parser class as represented by a basic state
414 machine. To create a specialized parser, make a subclass and override
415 any of the methods you care about. Methods that don't need to do
416 anything may just pass. However, _stateMachine should never be
417 overridden.
419 def __init__ (self, handler, followIncludes=True, errorsAreFatal=True,
420 missingIncludeIsFatal=True):
421 """Create a new KickstartParser instance. Instance attributes:
423 errorsAreFatal -- Should errors cause processing to halt, or
424 just print a message to the screen? This
425 is most useful for writing syntax checkers
426 that may want to continue after an error is
427 encountered.
428 followIncludes -- If %include is seen, should the included
429 file be checked as well or skipped?
430 handler -- An instance of a BaseHandler subclass. If
431 None, the input file will still be parsed
432 but no data will be saved and no commands
433 will be executed.
434 missingIncludeIsFatal -- Should missing include files be fatal, even
435 if errorsAreFatal is False?
437 self.errorsAreFatal = errorsAreFatal
438 self.followIncludes = followIncludes
439 self.handler = handler
440 self.currentdir = {}
441 self.missingIncludeIsFatal = missingIncludeIsFatal
443 self._state = STATE_COMMANDS
444 self._includeDepth = 0
445 self._line = ""
447 self.version = self.handler.version
449 global ver
450 ver = self.version
452 self._sections = {}
453 self.setupSections()
455 def _reset(self):
456 """Reset the internal variables of the state machine for a new kickstart file."""
457 self._state = STATE_COMMANDS
458 self._includeDepth = 0
460 def getSection(self, s):
461 """Return a reference to the requested section (s must start with '%'s),
462 or raise KeyError if not found.
464 return self._sections[s]
466 def handleCommand (self, lineno, args):
467 """Given the list of command and arguments, call the Version's
468 dispatcher method to handle the command. Returns the command or
469 data object returned by the dispatcher. This method may be
470 overridden in a subclass if necessary.
472 if self.handler:
473 self.handler.currentCmd = args[0]
474 self.handler.currentLine = self._line
475 retval = self.handler.dispatcher(args, lineno)
477 return retval
479 def registerSection(self, obj):
480 """Given an instance of a Section subclass, register the new section
481 with the parser. Calling this method means the parser will
482 recognize your new section and dispatch into the given object to
483 handle it.
485 if not obj.sectionOpen:
486 raise TypeError, "no sectionOpen given for section %s" % obj
488 if not obj.sectionOpen.startswith("%"):
489 raise TypeError, "section %s tag does not start with a %%" % obj.sectionOpen
491 self._sections[obj.sectionOpen] = obj
493 def _finalize(self, obj):
494 """Called at the close of a kickstart section to take any required
495 actions. Internally, this is used to add scripts once we have the
496 whole body read.
498 obj.finalize()
499 self._state = STATE_COMMANDS
501 def _handleSpecialComments(self, line):
502 """Kickstart recognizes a couple special comments."""
503 if self._state != STATE_COMMANDS:
504 return
506 # Save the platform for s-c-kickstart.
507 if line[:10] == "#platform=":
508 self.handler.platform = self._line[11:]
510 def _readSection(self, lineIter, lineno):
511 obj = self._sections[self._state]
513 while True:
514 try:
515 line = lineIter.next()
516 if line == "":
517 # This section ends at the end of the file.
518 if self.version >= version.F8:
519 raise KickstartParseError, formatErrorMsg(lineno, msg=_("Section does not end with %%end."))
521 self._finalize(obj)
522 except StopIteration:
523 break
525 lineno += 1
527 # Throw away blank lines and comments, unless the section wants all
528 # lines.
529 if self._isBlankOrComment(line) and not obj.allLines:
530 continue
532 if line.startswith("%"):
533 args = shlex.split(line)
535 if args and args[0] == "%end":
536 # This is a properly terminated section.
537 self._finalize(obj)
538 break
539 elif args and args[0] == "%ksappend":
540 continue
541 elif args and (self._validState(args[0]) or args[0] in ["%include", "%ksappend"]):
542 # This is an unterminated section.
543 if self.version >= version.F8:
544 raise KickstartParseError, formatErrorMsg(lineno, msg=_("Section does not end with %%end."))
546 # Finish up. We do not process the header here because
547 # kicking back out to STATE_COMMANDS will ensure that happens.
548 lineIter.put(line)
549 lineno -= 1
550 self._finalize(obj)
551 break
552 else:
553 # This is just a line within a section. Pass it off to whatever
554 # section handles it.
555 obj.handleLine(line)
557 return lineno
559 def _validState(self, st):
560 """Is the given section tag one that has been registered with the parser?"""
561 return st in self._sections.keys()
563 def _tryFunc(self, fn):
564 """Call the provided function (which doesn't take any arguments) and
565 do the appropriate error handling. If errorsAreFatal is False, this
566 function will just print the exception and keep going.
568 try:
569 fn()
570 except Exception as msg:
571 if self.errorsAreFatal:
572 raise
573 else:
574 print msg
576 def _isBlankOrComment(self, line):
577 return line.isspace() or line == "" or line.lstrip()[0] == '#'
579 def _stateMachine(self, lineIter):
580 # For error reporting.
581 lineno = 0
583 while True:
584 # Get the next line out of the file, quitting if this is the last line.
585 try:
586 self._line = lineIter.next()
587 if self._line == "":
588 break
589 except StopIteration:
590 break
592 lineno += 1
594 # Eliminate blank lines, whitespace-only lines, and comments.
595 if self._isBlankOrComment(self._line):
596 self._handleSpecialComments(self._line)
597 continue
599 # Remove any end-of-line comments.
600 sanitized = self._line.split("#")[0]
602 # Then split the line.
603 args = shlex.split(sanitized.rstrip())
605 if args[0] == "%include":
606 # This case comes up primarily in ksvalidator.
607 if not self.followIncludes:
608 continue
610 if len(args) == 1 or not args[1]:
611 raise KickstartParseError, formatErrorMsg(lineno)
613 self._includeDepth += 1
615 try:
616 self.readKickstart(args[1], reset=False)
617 except KickstartError:
618 # Handle the include file being provided over the
619 # network in a %pre script. This case comes up in the
620 # early parsing in anaconda.
621 if self.missingIncludeIsFatal:
622 raise
624 self._includeDepth -= 1
625 continue
627 # Now on to the main event.
628 if self._state == STATE_COMMANDS:
629 if args[0] == "%ksappend":
630 # This is handled by the preprocess* functions, so continue.
631 continue
632 elif args[0][0] == '%':
633 # This is the beginning of a new section. Handle its header
634 # here.
635 newSection = args[0]
636 if not self._validState(newSection):
637 raise KickstartParseError, formatErrorMsg(lineno, msg=_("Unknown kickstart section: %s" % newSection))
639 self._state = newSection
640 obj = self._sections[self._state]
641 self._tryFunc(lambda: obj.handleHeader(lineno, args))
643 # This will handle all section processing, kicking us back
644 # out to STATE_COMMANDS at the end with the current line
645 # being the next section header, etc.
646 lineno = self._readSection(lineIter, lineno)
647 else:
648 # This is a command in the command section. Dispatch to it.
649 self._tryFunc(lambda: self.handleCommand(lineno, args))
650 elif self._state == STATE_END:
651 break
653 def readKickstartFromString (self, s, reset=True):
654 """Process a kickstart file, provided as the string str."""
655 if reset:
656 self._reset()
658 # Add a "" to the end of the list so the string reader acts like the
659 # file reader and we only get StopIteration when we're after the final
660 # line of input.
661 i = PutBackIterator(s.splitlines(True) + [""])
662 self._stateMachine (i)
664 def readKickstart(self, f, reset=True):
665 """Process a kickstart file, given by the filename f."""
666 if reset:
667 self._reset()
669 # an %include might not specify a full path. if we don't try to figure
670 # out what the path should have been, then we're unable to find it
671 # requiring full path specification, though, sucks. so let's make
672 # the reading "smart" by keeping track of what the path is at each
673 # include depth.
674 if not os.path.exists(f):
675 if self.currentdir.has_key(self._includeDepth - 1):
676 if os.path.exists(os.path.join(self.currentdir[self._includeDepth - 1], f)):
677 f = os.path.join(self.currentdir[self._includeDepth - 1], f)
679 cd = os.path.dirname(f)
680 if not cd.startswith("/"):
681 cd = os.path.abspath(cd)
682 self.currentdir[self._includeDepth] = cd
684 try:
685 s = urlread(f)
686 except grabber.URLGrabError, e:
687 raise KickstartError, formatErrorMsg(0, msg=_("Unable to open input kickstart file: %s") % e.strerror)
689 self.readKickstartFromString(s, reset=False)
691 def setupSections(self):
692 """Install the sections all kickstart files support. You may override
693 this method in a subclass, but should avoid doing so unless you know
694 what you're doing.
696 self._sections = {}
698 # Install the sections all kickstart files support.
699 self.registerSection(PreScriptSection(self.handler, dataObj=Script))
700 self.registerSection(PostScriptSection(self.handler, dataObj=Script))
701 self.registerSection(TracebackScriptSection(self.handler, dataObj=Script))
702 self.registerSection(PackageSection(self.handler))