Add --install flag to repo command (#1119867)
[pykickstart.git] / pykickstart / parser.py
blob94b21bb018d92212f96760dce624e2e24f8a233c
2 # parser.py: Kickstart file parser.
4 # Chris Lumens <clumens@redhat.com>
6 # Copyright 2005, 2006, 2007, 2008, 2011 Red Hat, Inc.
8 # This copyrighted material is made available to anyone wishing to use, modify,
9 # copy, or redistribute it subject to the terms and conditions of the GNU
10 # General Public License v.2. This program is distributed in the hope that it
11 # will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
12 # implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 # See the GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along with
16 # this program; if not, write to the Free Software Foundation, Inc., 51
17 # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
18 # trademarks that are incorporated in the source code or documentation are not
19 # subject to the GNU General Public License and may only be used or replicated
20 # with the express permission of Red Hat, Inc.
22 """
23 Main kickstart file processing module.
25 This module exports several important classes:
27 Script - Representation of a single %pre, %post, or %traceback script.
29 Packages - Representation of the %packages section.
31 KickstartParser - The kickstart file parser state machine.
32 """
34 from collections import Iterator
35 import os
36 import shlex
37 import sys
38 import tempfile
39 from copy import copy
40 from optparse import *
41 from urlgrabber import urlread
42 import urlgrabber.grabber as grabber
44 import constants
45 from errors import KickstartError, KickstartParseError, KickstartValueError, formatErrorMsg
46 from ko import KickstartObject
47 from sections import *
48 import version
50 import gettext
51 _ = lambda x: gettext.ldgettext("pykickstart", x)
53 STATE_END = "end"
54 STATE_COMMANDS = "commands"
56 ver = version.DEVEL
58 def _preprocessStateMachine (lineIter):
59 l = None
60 lineno = 0
62 # Now open an output kickstart file that we are going to write to one
63 # line at a time.
64 (outF, outName) = tempfile.mkstemp("-ks.cfg", "", "/tmp")
66 while True:
67 try:
68 l = lineIter.next()
69 except StopIteration:
70 break
72 # At the end of the file?
73 if l == "":
74 break
76 lineno += 1
77 url = None
79 ll = l.strip()
80 if not ll.startswith("%ksappend"):
81 os.write(outF, l)
82 continue
84 # Try to pull down the remote file.
85 try:
86 ksurl = ll.split(' ')[1]
87 except:
88 raise KickstartParseError(formatErrorMsg(lineno, msg=_("Illegal url for %%ksappend: %s") % ll))
90 try:
91 url = grabber.urlopen(ksurl)
92 except grabber.URLGrabError, e:
93 raise KickstartError(formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file: %s") % e.strerror))
94 else:
95 # Sanity check result. Sometimes FTP doesn't catch a file
96 # is missing.
97 try:
98 if url.size < 1:
99 raise KickstartError(formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file")))
100 except:
101 raise KickstartError(formatErrorMsg(lineno, msg=_("Unable to open %%ksappend file")))
103 # If that worked, write the remote file to the output kickstart
104 # file in one burst. Then close everything up to get ready to
105 # read ahead in the input file. This allows multiple %ksappend
106 # lines to exist.
107 if url is not None:
108 os.write(outF, url.read())
109 url.close()
111 # All done - close the temp file and return its location.
112 os.close(outF)
113 return outName
115 def preprocessFromString (s):
116 """Preprocess the kickstart file, provided as the string str. This
117 method is currently only useful for handling %ksappend lines,
118 which need to be fetched before the real kickstart parser can be
119 run. Returns the location of the complete kickstart file.
121 i = iter(s.splitlines(True) + [""])
122 rc = _preprocessStateMachine (i.next)
123 return rc
125 def preprocessKickstart (f):
126 """Preprocess the kickstart file, given by the filename file. This
127 method is currently only useful for handling %ksappend lines,
128 which need to be fetched before the real kickstart parser can be
129 run. Returns the location of the complete kickstart file.
131 try:
132 fh = urlopen(f)
133 except grabber.URLGrabError, e:
134 raise KickstartError(formatErrorMsg(0, msg=_("Unable to open input kickstart file: %s") % e.strerror))
136 rc = _preprocessStateMachine (iter(fh.readlines()))
137 fh.close()
138 return rc
140 class PutBackIterator(Iterator):
141 def __init__(self, iterable):
142 self._iterable = iter(iterable)
143 self._buf = None
145 def __iter__(self):
146 return self
148 def put(self, s):
149 self._buf = s
151 def next(self):
152 if self._buf:
153 retval = self._buf
154 self._buf = None
155 return retval
156 else:
157 return self._iterable.next()
160 ### SCRIPT HANDLING
162 class Script(KickstartObject):
163 """A class representing a single kickstart script. If functionality beyond
164 just a data representation is needed (for example, a run method in
165 anaconda), Script may be subclassed. Although a run method is not
166 provided, most of the attributes of Script have to do with running the
167 script. Instances of Script are held in a list by the Version object.
169 def __init__(self, script, *args , **kwargs):
170 """Create a new Script instance. Instance attributes:
172 errorOnFail -- If execution of the script fails, should anaconda
173 stop, display an error, and then reboot without
174 running any other scripts?
175 inChroot -- Does the script execute in anaconda's chroot
176 environment or not?
177 interp -- The program that should be used to interpret this
178 script.
179 lineno -- The line number this script starts on.
180 logfile -- Where all messages from the script should be logged.
181 script -- A string containing all the lines of the script.
182 type -- The type of the script, which can be KS_SCRIPT_* from
183 pykickstart.constants.
185 KickstartObject.__init__(self, *args, **kwargs)
186 self.script = "".join(script)
188 self.interp = kwargs.get("interp", "/bin/sh")
189 self.inChroot = kwargs.get("inChroot", False)
190 self.lineno = kwargs.get("lineno", None)
191 self.logfile = kwargs.get("logfile", None)
192 self.errorOnFail = kwargs.get("errorOnFail", False)
193 self.type = kwargs.get("type", constants.KS_SCRIPT_PRE)
195 def __str__(self):
196 """Return a string formatted for output to a kickstart file."""
197 retval = ""
199 if self.type == constants.KS_SCRIPT_PRE:
200 retval += '\n%pre'
201 elif self.type == constants.KS_SCRIPT_POST:
202 retval += '\n%post'
203 elif self.type == constants.KS_SCRIPT_TRACEBACK:
204 retval += '\n%traceback'
206 if self.interp != "/bin/sh" and self.interp != "":
207 retval += " --interpreter=%s" % self.interp
208 if self.type == constants.KS_SCRIPT_POST and not self.inChroot:
209 retval += " --nochroot"
210 if self.logfile != None:
211 retval += " --logfile %s" % self.logfile
212 if self.errorOnFail:
213 retval += " --erroronfail"
215 if self.script.endswith("\n"):
216 if ver >= version.F8:
217 return retval + "\n%s%%end\n" % self.script
218 else:
219 return retval + "\n%s\n" % self.script
220 else:
221 if ver >= version.F8:
222 return retval + "\n%s\n%%end\n" % self.script
223 else:
224 return retval + "\n%s\n" % self.script
228 ## PACKAGE HANDLING
230 class Group:
231 """A class representing a single group in the %packages section."""
232 def __init__(self, name="", include=constants.GROUP_DEFAULT):
233 """Create a new Group instance. Instance attributes:
235 name -- The group's identifier
236 include -- The level of how much of the group should be included.
237 Values can be GROUP_* from pykickstart.constants.
239 self.name = name
240 self.include = include
242 def __str__(self):
243 """Return a string formatted for output to a kickstart file."""
244 if self.include == constants.GROUP_REQUIRED:
245 return "@%s --nodefaults" % self.name
246 elif self.include == constants.GROUP_ALL:
247 return "@%s --optional" % self.name
248 else:
249 return "@%s" % self.name
251 def __cmp__(self, other):
252 if self.name < other.name:
253 return -1
254 elif self.name > other.name:
255 return 1
256 return 0
258 class Packages(KickstartObject):
259 """A class representing the %packages section of the kickstart file."""
260 def __init__(self, *args, **kwargs):
261 """Create a new Packages instance. Instance attributes:
263 addBase -- Should the Base group be installed even if it is
264 not specified?
265 default -- Should the default package set be selected?
266 environment -- What base environment should be selected? Only one
267 may be chosen at a time.
268 excludedList -- A list of all the packages marked for exclusion in
269 the %packages section, without the leading minus
270 symbol.
271 excludeDocs -- Should documentation in each package be excluded?
272 groupList -- A list of Group objects representing all the groups
273 specified in the %packages section. Names will be
274 stripped of the leading @ symbol.
275 excludedGroupList -- A list of Group objects representing all the
276 groups specified for removal in the %packages
277 section. Names will be stripped of the leading
278 -@ symbols.
279 handleMissing -- If unknown packages are specified in the %packages
280 section, should it be ignored or not? Values can
281 be KS_MISSING_* from pykickstart.constants.
282 packageList -- A list of all the packages specified in the
283 %packages section.
284 instLangs -- A list of languages to install.
285 multiLib -- Whether to use yum's "all" multilib policy.
286 seen -- If %packages was ever used in the kickstart file,
287 this attribute will be set to True.
290 KickstartObject.__init__(self, *args, **kwargs)
292 self.addBase = True
293 self.default = False
294 self.environment = None
295 self.excludedList = []
296 self.excludedGroupList = []
297 self.excludeDocs = False
298 self.groupList = []
299 self.handleMissing = constants.KS_MISSING_PROMPT
300 self.packageList = []
301 self.instLangs = None
302 self.multiLib = False
303 self.seen = False
305 def __str__(self):
306 """Return a string formatted for output to a kickstart file."""
307 pkgs = ""
309 if not self.default:
310 if self.environment:
311 pkgs += "@^%s\n" % self.environment
313 grps = self.groupList
314 grps.sort()
315 for grp in grps:
316 pkgs += "%s\n" % grp.__str__()
318 p = self.packageList
319 p.sort()
320 for pkg in p:
321 pkgs += "%s\n" % pkg
323 grps = self.excludedGroupList
324 grps.sort()
325 for grp in grps:
326 pkgs += "-%s\n" % grp.__str__()
328 p = self.excludedList
329 p.sort()
330 for pkg in p:
331 pkgs += "-%s\n" % pkg
333 if pkgs == "":
334 return ""
336 retval = "\n%packages"
338 if self.default:
339 retval += " --default"
340 if self.excludeDocs:
341 retval += " --excludedocs"
342 if not self.addBase:
343 retval += " --nobase"
344 if self.handleMissing == constants.KS_MISSING_IGNORE:
345 retval += " --ignoremissing"
346 if self.instLangs:
347 retval += " --instLangs=%s" % self.instLangs
348 if self.multiLib:
349 retval += " --multilib"
351 if ver >= version.F8:
352 return retval + "\n" + pkgs + "\n%end\n"
353 else:
354 return retval + "\n" + pkgs + "\n"
356 def _processGroup (self, line):
357 op = OptionParser()
358 op.add_option("--nodefaults", action="store_true", default=False)
359 op.add_option("--optional", action="store_true", default=False)
361 (opts, extra) = op.parse_args(args=line.split())
363 if opts.nodefaults and opts.optional:
364 raise KickstartValueError(_("Group cannot specify both --nodefaults and --optional"))
366 # If the group name has spaces in it, we have to put it back together
367 # now.
368 grp = " ".join(extra)
370 if opts.nodefaults:
371 self.groupList.append(Group(name=grp, include=constants.GROUP_REQUIRED))
372 elif opts.optional:
373 self.groupList.append(Group(name=grp, include=constants.GROUP_ALL))
374 else:
375 self.groupList.append(Group(name=grp, include=constants.GROUP_DEFAULT))
377 def add (self, pkgList):
378 """Given a list of lines from the input file, strip off any leading
379 symbols and add the result to the appropriate list.
381 existingExcludedSet = set(self.excludedList)
382 existingPackageSet = set(self.packageList)
383 newExcludedSet = set()
384 newPackageSet = set()
386 excludedGroupList = []
388 for pkg in pkgList:
389 stripped = pkg.strip()
391 if stripped[0:2] == "@^":
392 self.environment = stripped[2:]
393 elif stripped[0] == "@":
394 self._processGroup(stripped[1:])
395 elif stripped[0] == "-":
396 if stripped[1:3] == "@^" and self.environment == stripped[3:]:
397 self.environment = None
398 elif stripped[1] == "@":
399 excludedGroupList.append(Group(name=stripped[2:]))
400 else:
401 newExcludedSet.add(stripped[1:])
402 else:
403 newPackageSet.add(stripped)
405 # Groups have to be excluded in two different ways (note: can't use
406 # sets here because we have to store objects):
407 excludedGroupNames = map(lambda g: g.name, excludedGroupList)
409 # First, an excluded group may be cancelling out a previously given
410 # one. This is often the case when using %include. So there we should
411 # just remove the group from the list.
412 self.groupList = filter(lambda g: g.name not in excludedGroupNames, self.groupList)
414 # Second, the package list could have included globs which are not
415 # processed by pykickstart. In that case we need to preserve a list of
416 # excluded groups so whatever tool doing package/group installation can
417 # take appropriate action.
418 self.excludedGroupList.extend(excludedGroupList)
420 existingPackageSet = (existingPackageSet - newExcludedSet) | newPackageSet
421 existingExcludedSet = (existingExcludedSet - existingPackageSet) | newExcludedSet
423 self.packageList = list(existingPackageSet)
424 self.excludedList = list(existingExcludedSet)
428 ### PARSER
430 class KickstartParser:
431 """The kickstart file parser class as represented by a basic state
432 machine. To create a specialized parser, make a subclass and override
433 any of the methods you care about. Methods that don't need to do
434 anything may just pass. However, _stateMachine should never be
435 overridden.
437 def __init__ (self, handler, followIncludes=True, errorsAreFatal=True,
438 missingIncludeIsFatal=True):
439 """Create a new KickstartParser instance. Instance attributes:
441 errorsAreFatal -- Should errors cause processing to halt, or
442 just print a message to the screen? This
443 is most useful for writing syntax checkers
444 that may want to continue after an error is
445 encountered.
446 followIncludes -- If %include is seen, should the included
447 file be checked as well or skipped?
448 handler -- An instance of a BaseHandler subclass. If
449 None, the input file will still be parsed
450 but no data will be saved and no commands
451 will be executed.
452 missingIncludeIsFatal -- Should missing include files be fatal, even
453 if errorsAreFatal is False?
455 self.errorsAreFatal = errorsAreFatal
456 self.followIncludes = followIncludes
457 self.handler = handler
458 self.currentdir = {}
459 self.missingIncludeIsFatal = missingIncludeIsFatal
461 self._state = STATE_COMMANDS
462 self._includeDepth = 0
463 self._line = ""
465 self.version = self.handler.version
467 global ver
468 ver = self.version
470 self._sections = {}
471 self.setupSections()
473 def _reset(self):
474 """Reset the internal variables of the state machine for a new kickstart file."""
475 self._state = STATE_COMMANDS
476 self._includeDepth = 0
478 def getSection(self, s):
479 """Return a reference to the requested section (s must start with '%'s),
480 or raise KeyError if not found.
482 return self._sections[s]
484 def handleCommand (self, lineno, args):
485 """Given the list of command and arguments, call the Version's
486 dispatcher method to handle the command. Returns the command or
487 data object returned by the dispatcher. This method may be
488 overridden in a subclass if necessary.
490 if self.handler:
491 self.handler.currentCmd = args[0]
492 self.handler.currentLine = self._line
493 retval = self.handler.dispatcher(args, lineno)
495 return retval
497 def registerSection(self, obj):
498 """Given an instance of a Section subclass, register the new section
499 with the parser. Calling this method means the parser will
500 recognize your new section and dispatch into the given object to
501 handle it.
503 if not obj.sectionOpen:
504 raise TypeError("no sectionOpen given for section %s" % obj)
506 if not obj.sectionOpen.startswith("%"):
507 raise TypeError("section %s tag does not start with a %%" % obj.sectionOpen)
509 self._sections[obj.sectionOpen] = obj
511 def _finalize(self, obj):
512 """Called at the close of a kickstart section to take any required
513 actions. Internally, this is used to add scripts once we have the
514 whole body read.
516 obj.finalize()
517 self._state = STATE_COMMANDS
519 def _handleSpecialComments(self, line):
520 """Kickstart recognizes a couple special comments."""
521 if self._state != STATE_COMMANDS:
522 return
524 # Save the platform for s-c-kickstart.
525 if line[:10] == "#platform=":
526 self.handler.platform = self._line[11:]
528 def _readSection(self, lineIter, lineno):
529 obj = self._sections[self._state]
531 while True:
532 try:
533 line = lineIter.next()
534 if line == "" and self._includeDepth == 0:
535 # This section ends at the end of the file.
536 if self.version >= version.F8:
537 raise KickstartParseError(formatErrorMsg(lineno, msg=_("Section %s does not end with %%end.") % obj.sectionOpen))
539 self._finalize(obj)
540 except StopIteration:
541 break
543 lineno += 1
545 # Throw away blank lines and comments, unless the section wants all
546 # lines.
547 if self._isBlankOrComment(line) and not obj.allLines:
548 continue
550 if line.startswith("%"):
551 # If we're in a script, the line may begin with "%something"
552 # that's not the start of any section we recognize, but still
553 # valid for that script. So, don't do the split below unless
554 # we're sure.
555 possibleSectionStart = line.split()[0]
556 if not self._validState(possibleSectionStart) \
557 and possibleSectionStart not in ("%end", "%include"):
558 obj.handleLine(line)
559 continue
561 args = shlex.split(line)
563 if args and args[0] == "%end":
564 # This is a properly terminated section.
565 self._finalize(obj)
566 break
567 elif args and args[0] == "%include":
568 if len(args) == 1 or not args[1]:
569 raise KickstartParseError(formatErrorMsg(lineno))
571 self._handleInclude(args[1])
572 continue
573 elif args and args[0] == "%ksappend":
574 continue
575 elif args and self._validState(args[0]):
576 # This is an unterminated section.
577 if self.version >= version.F8:
578 raise KickstartParseError(formatErrorMsg(lineno, msg=_("Section %s does not end with %%end.") % obj.sectionOpen))
580 # Finish up. We do not process the header here because
581 # kicking back out to STATE_COMMANDS will ensure that happens.
582 lineIter.put(line)
583 lineno -= 1
584 self._finalize(obj)
585 break
586 else:
587 # This is just a line within a section. Pass it off to whatever
588 # section handles it.
589 obj.handleLine(line)
591 return lineno
593 def _validState(self, st):
594 """Is the given section tag one that has been registered with the parser?"""
595 return st in self._sections.keys()
597 def _tryFunc(self, fn):
598 """Call the provided function (which doesn't take any arguments) and
599 do the appropriate error handling. If errorsAreFatal is False, this
600 function will just print the exception and keep going.
602 try:
603 fn()
604 except Exception, msg:
605 if self.errorsAreFatal:
606 raise
607 else:
608 print msg
610 def _isBlankOrComment(self, line):
611 return line.isspace() or line == "" or line.lstrip()[0] == '#'
613 def _handleInclude(self, f):
614 # This case comes up primarily in ksvalidator.
615 if not self.followIncludes:
616 return
618 self._includeDepth += 1
620 try:
621 self.readKickstart(f, reset=False)
622 except KickstartError:
623 # Handle the include file being provided over the
624 # network in a %pre script. This case comes up in the
625 # early parsing in anaconda.
626 if self.missingIncludeIsFatal:
627 raise
629 self._includeDepth -= 1
631 def _stateMachine(self, lineIter):
632 # For error reporting.
633 lineno = 0
635 while True:
636 # Get the next line out of the file, quitting if this is the last line.
637 try:
638 self._line = lineIter.next()
639 if self._line == "":
640 break
641 except StopIteration:
642 break
644 lineno += 1
646 # Eliminate blank lines, whitespace-only lines, and comments.
647 if self._isBlankOrComment(self._line):
648 self._handleSpecialComments(self._line)
649 continue
651 # Split the line, discarding comments.
652 args = shlex.split(self._line, comments=True)
654 if args[0] == "%include":
655 if len(args) == 1 or not args[1]:
656 raise KickstartParseError(formatErrorMsg(lineno))
658 self._handleInclude(args[1])
659 continue
661 # Now on to the main event.
662 if self._state == STATE_COMMANDS:
663 if args[0] == "%ksappend":
664 # This is handled by the preprocess* functions, so continue.
665 continue
666 elif args[0][0] == '%':
667 # This is the beginning of a new section. Handle its header
668 # here.
669 newSection = args[0]
670 if not self._validState(newSection):
671 raise KickstartParseError(formatErrorMsg(lineno, msg=_("Unknown kickstart section: %s" % newSection)))
673 self._state = newSection
674 obj = self._sections[self._state]
675 self._tryFunc(lambda: obj.handleHeader(lineno, args))
677 # This will handle all section processing, kicking us back
678 # out to STATE_COMMANDS at the end with the current line
679 # being the next section header, etc.
680 lineno = self._readSection(lineIter, lineno)
681 else:
682 # This is a command in the command section. Dispatch to it.
683 self._tryFunc(lambda: self.handleCommand(lineno, args))
684 elif self._state == STATE_END:
685 break
686 elif self._includeDepth > 0:
687 lineIter.put(self._line)
688 lineno -= 1
689 lineno = self._readSection(lineIter, lineno)
691 def readKickstartFromString (self, s, reset=True):
692 """Process a kickstart file, provided as the string str."""
693 if reset:
694 self._reset()
696 # Add a "" to the end of the list so the string reader acts like the
697 # file reader and we only get StopIteration when we're after the final
698 # line of input.
699 i = PutBackIterator(s.splitlines(True) + [""])
700 self._stateMachine (i)
702 def readKickstart(self, f, reset=True):
703 """Process a kickstart file, given by the filename f."""
704 if reset:
705 self._reset()
707 # an %include might not specify a full path. if we don't try to figure
708 # out what the path should have been, then we're unable to find it
709 # requiring full path specification, though, sucks. so let's make
710 # the reading "smart" by keeping track of what the path is at each
711 # include depth.
712 if not os.path.exists(f):
713 if self.currentdir.has_key(self._includeDepth - 1):
714 if os.path.exists(os.path.join(self.currentdir[self._includeDepth - 1], f)):
715 f = os.path.join(self.currentdir[self._includeDepth - 1], f)
717 cd = os.path.dirname(f)
718 if not cd.startswith("/"):
719 cd = os.path.abspath(cd)
720 self.currentdir[self._includeDepth] = cd
722 try:
723 s = urlread(f)
724 except grabber.URLGrabError, e:
725 raise KickstartError(formatErrorMsg(0, msg=_("Unable to open input kickstart file: %s") % e.strerror))
727 self.readKickstartFromString(s, reset=False)
729 def setupSections(self):
730 """Install the sections all kickstart files support. You may override
731 this method in a subclass, but should avoid doing so unless you know
732 what you're doing.
734 self._sections = {}
736 # Install the sections all kickstart files support.
737 self.registerSection(PreScriptSection(self.handler, dataObj=Script))
738 self.registerSection(PostScriptSection(self.handler, dataObj=Script))
739 self.registerSection(TracebackScriptSection(self.handler, dataObj=Script))
740 self.registerSection(PackageSection(self.handler))