tools/python/pkglint/userland.py

   1 #!/usr/bin/python
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22
  23 #
  24 # Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
  25 #
  26
  27 # Some userland consolidation specific lint checks
  28
  29 import pkg.lint.base as base
  30 from pkg.lint.engine import lint_fmri_successor
  31 import pkg.fmri
  32 import pkg.elf as elf
  33 import re
  34 import os.path
  35 import subprocess
  36 import pkg.client.api
  37 import pkg.client.api_errors
  38 import pkg.client.progress
  39
  40 class UserlandActionChecker(base.ActionChecker):
  41         """An opensolaris.org-specific class to check actions."""
  42
  43         name = "userland.action"
  44
  45         def __init__(self, config):
  46                 self.description = _(
  47                     "checks Userland packages for common content errors")
  48                 path = os.getenv('PROTO_PATH')
  49                 if path != None:
  50                         self.proto_path = path.split()
  51                 else:
  52                         self.proto_path = None
  53                 #
  54                 # These lists are used to check if a 32/64-bit binary
  55                 # is in a proper 32/64-bit directory.
  56                 #
  57                 self.pathlist32 = [
  58                         "i86",
  59                         "sparcv7",
  60                         "32",
  61                         "i86pc-solaris-64int",  # perl path
  62                         "sun4-solaris-64int"    # perl path
  63                 ]
  64                 self.pathlist64 = [
  65                         "amd64",
  66                         "sparcv9",
  67                         "64",
  68                         "i86pc-solaris-64",     # perl path
  69                         "sun4-solaris-64"       # perl path
  70                 ]
  71                 self.runpath_re = [
  72                         re.compile('^/lib(/.*)?$'),
  73                         re.compile('^/usr/'),
  74                         re.compile('^\$ORIGIN/')
  75                 ]
  76                 self.runpath_64_re = [
  77                         re.compile('^.*/64(/.*)?$'),
  78                         re.compile('^.*/amd64(/.*)?$'),
  79                         re.compile('^.*/sparcv9(/.*)?$'),
  80                         re.compile('^.*/i86pc-solaris-64(/.*)?$'), # perl path
  81                         re.compile('^.*/sun4-solaris-64(/.*)?$')   # perl path
  82                 ]
  83                 self.initscript_re = re.compile("^etc/(rc.|init)\.d")
  84
  85                 self.lint_paths = {}
  86                 self.ref_paths = {}
  87
  88                 super(UserlandActionChecker, self).__init__(config)
  89
  90         def startup(self, engine):
  91                 """Initialize the checker with a dictionary of paths, so that we
  92                 can do link resolution.
  93
  94                 This is copied from the core pkglint code, but should eventually
  95                 be made common.
  96                 """
  97
  98                 def seed_dict(mf, attr, dic, atype=None, verbose=False):
  99                         """Updates a dictionary of { attr: [(fmri, action), ..]}
 100                         where attr is the value of that attribute from
 101                         actions of a given type atype, in the given
 102                         manifest."""
 103
 104                         pkg_vars = mf.get_all_variants()
 105
 106                         if atype:
 107                                 mfg = (a for a in mf.gen_actions_by_type(atype))
 108                         else:
 109                                 mfg = (a for a in mf.gen_actions())
 110
 111                         for action in mfg:
 112                                 if atype and action.name != atype:
 113                                         continue
 114                                 if attr not in action.attrs:
 115                                         continue
 116
 117                                 variants = action.get_variant_template()
 118                                 variants.merge_unknown(pkg_vars)
 119                                 action.attrs.update(variants)
 120
 121                                 p = action.attrs[attr]
 122                                 dic.setdefault(p, []).append((mf.fmri, action))
 123
 124                 # construct a set of FMRIs being presented for linting, and
 125                 # avoid seeding the reference dictionary with any for which
 126                 # we're delivering new packages.
 127                 lint_fmris = {}
 128                 for m in engine.gen_manifests(engine.lint_api_inst,
 129                     release=engine.release, pattern=engine.pattern):
 130                         lint_fmris.setdefault(m.fmri.get_name(), []).append(m.fmri)
 131                 for m in engine.lint_manifests:
 132                         lint_fmris.setdefault(m.fmri.get_name(), []).append(m.fmri)
 133
 134                 engine.logger.debug(
 135                     _("Seeding reference action path dictionaries."))
 136
 137                 for manifest in engine.gen_manifests(engine.ref_api_inst,
 138                     release=engine.release):
 139                         # Only put this manifest into the reference dictionary
 140                         # if it's not an older version of the same package.
 141                         if not any(
 142                             lint_fmri_successor(fmri, manifest.fmri)
 143                             for fmri
 144                             in lint_fmris.get(manifest.fmri.get_name(), [])
 145                         ):
 146                                 seed_dict(manifest, "path", self.ref_paths)
 147
 148                 engine.logger.debug(
 149                     _("Seeding lint action path dictionaries."))
 150
 151                 # we provide a search pattern, to allow users to lint a
 152                 # subset of the packages in the lint_repository
 153                 for manifest in engine.gen_manifests(engine.lint_api_inst,
 154                     release=engine.release, pattern=engine.pattern):
 155                         seed_dict(manifest, "path", self.lint_paths)
 156
 157                 engine.logger.debug(
 158                     _("Seeding local action path dictionaries."))
 159
 160                 for manifest in engine.lint_manifests:
 161                         seed_dict(manifest, "path", self.lint_paths)
 162
 163                 self.__merge_dict(self.lint_paths, self.ref_paths,
 164                     ignore_pubs=engine.ignore_pubs)
 165
 166         def __merge_dict(self, src, target, ignore_pubs=True):
 167                 """Merges the given src dictionary into the target
 168                 dictionary, giving us the target content as it would appear,
 169                 were the packages in src to get published to the
 170                 repositories that made up target.
 171
 172                 We need to only merge packages at the same or successive
 173                 version from the src dictionary into the target dictionary.
 174                 If the src dictionary contains a package with no version
 175                 information, it is assumed to be more recent than the same
 176                 package with no version in the target."""
 177
 178                 for p in src:
 179                         if p not in target:
 180                                 target[p] = src[p]
 181                                 continue
 182
 183                         def build_dic(arr):
 184                                 """Builds a dictionary of fmri:action entries"""
 185                                 dic = {}
 186                                 for (pfmri, action) in arr:
 187                                         if pfmri in dic:
 188                                                 dic[pfmri].append(action)
 189                                         else:
 190                                                 dic[pfmri] = [action]
 191                                 return dic
 192
 193                         src_dic = build_dic(src[p])
 194                         targ_dic = build_dic(target[p])
 195
 196                         for src_pfmri in src_dic:
 197                                 # we want to remove entries deemed older than
 198                                 # src_pfmri from targ_dic.
 199                                 for targ_pfmri in targ_dic.copy():
 200                                         sname = src_pfmri.get_name()
 201                                         tname = targ_pfmri.get_name()
 202                                         if lint_fmri_successor(src_pfmri,
 203                                             targ_pfmri,
 204                                             ignore_pubs=ignore_pubs):
 205                                                 targ_dic.pop(targ_pfmri)
 206                         targ_dic.update(src_dic)
 207                         l = []
 208                         for pfmri in targ_dic:
 209                                 for action in targ_dic[pfmri]:
 210                                         l.append((pfmri, action))
 211                         target[p] = l
 212
 213         def __realpath(self, path, target):
 214                 """Combine path and target to get the real path."""
 215
 216                 result = os.path.dirname(path)
 217
 218                 for frag in target.split(os.sep):
 219                         if frag == '..':
 220                                 result = os.path.dirname(result)
 221                         elif frag == '.':
 222                                 pass
 223                         else:
 224                                 result = os.path.join(result, frag)
 225
 226                 return result
 227
 228         def __elf_aslr_check(self, path, engine):
 229                 result = None
 230
 231                 ei = elf.get_info(path)
 232                 type = ei.get("type");
 233                 if type != "exe":
 234                         return result
 235
 236                 # get the ASLR tag string for this binary
 237                 aslr_tag_process = subprocess.Popen(
 238                         "/usr/bin/elfedit -r -e 'dyn:sunw_aslr' "
 239                         + path, shell=True,
 240                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 241
 242                 # aslr_tag_string will get stdout; err will get stderr
 243                 aslr_tag_string, err = aslr_tag_process.communicate()
 244
 245                 # No ASLR tag was found; everthing must be tagged
 246                 if aslr_tag_process.returncode != 0:
 247                         engine.error(
 248                                 _("'%s' is not tagged for aslr") % (path),
 249                                 msgid="%s%s.5" % (self.name, "001"))
 250                         return result
 251
 252                 # look for "ENABLE" anywhere in the string;
 253                 # warn about binaries which are not ASLR enabled
 254                 if re.search("ENABLE", aslr_tag_string) is not None:
 255                         return result
 256                 engine.warning(
 257                         _("'%s' does not have aslr enabled") % (path),
 258                         msgid="%s%s.6" % (self.name, "001"))
 259                 return result
 260
 261         def __elf_runpath_check(self, path, engine):
 262                 result = None
 263                 list = []
 264
 265                 ed = elf.get_dynamic(path)
 266                 ei = elf.get_info(path)
 267                 bits = ei.get("bits")
 268                 for dir in ed.get("runpath", "").split(":"):
 269                         if dir == None or dir == '':
 270                                 continue
 271
 272                         match = False
 273                         for expr in self.runpath_re:
 274                                 if expr.match(dir):
 275                                         match = True
 276                                         break
 277
 278                         if match == False:
 279                                 list.append(dir)
 280
 281                         if bits == 32:
 282                                 for expr in self.runpath_64_re:
 283                                         if expr.search(dir):
 284                                                 engine.warning(
 285                                                         _("64-bit runpath in 32-bit binary, '%s' includes '%s'") % (path, dir),
 286                                                         msgid="%s%s.3" % (self.name, "001"))
 287                         else:
 288                                 match = False
 289                                 for expr in self.runpath_64_re:
 290                                         if expr.search(dir):
 291                                                 match = True
 292                                                 break
 293                                 if match == False:
 294                                         engine.warning(
 295                                                 _("32-bit runpath in 64-bit binary, '%s' includes '%s'") % (path, dir),
 296                                                 msgid="%s%s.3" % (self.name, "001"))
 297                 if len(list) > 0:
 298                         result = _("bad RUNPATH, '%%s' includes '%s'" %
 299                                    ":".join(list))
 300
 301                 return result
 302
 303         def __elf_wrong_location_check(self, path):
 304                 result = None
 305
 306                 ei = elf.get_info(path)
 307                 bits = ei.get("bits")
 308                 type = ei.get("type");
 309                 elems = os.path.dirname(path).split("/")
 310
 311                 path64 = False
 312                 for p in self.pathlist64:
 313                     if (p in elems):
 314                         path64 = True
 315
 316                 path32 = False
 317                 for p in self.pathlist32:
 318                     if (p in elems):
 319                         path32 = True
 320
 321                 # ignore 64-bit executables in normal (non-32-bit-specific)
 322                 # locations, that's ok now.
 323                 if (type == "exe" and bits == 64 and path32 == False and path64 == False):
 324                         return result
 325
 326                 if bits == 32 and path64:
 327                         result = _("32-bit object '%s' in 64-bit path")
 328                 elif bits == 64 and not path64:
 329                         result = _("64-bit object '%s' in 32-bit path")
 330                 return result
 331
 332         def file_action(self, action, manifest, engine, pkglint_id="001"):
 333                 """Checks for existence in the proto area."""
 334
 335                 if action.name not in ["file"]:
 336                         return
 337
 338                 path = action.hash
 339                 if path == None or path == 'NOHASH':
 340                         path = action.attrs["path"]
 341
 342                 # check for writable files without a preserve attribute
 343                 if "mode" in action.attrs:
 344                         mode = action.attrs["mode"]
 345
 346                         if (int(mode, 8) & 0222) != 0 and "preserve" not in action.attrs:
 347                                 engine.error(
 348                                 _("%(path)s is writable (%(mode)s), but missing a preserve"
 349                                   " attribute") %  {"path": path, "mode": mode},
 350                                 msgid="%s%s.0" % (self.name, pkglint_id))
 351                 elif "preserve" in action.attrs:
 352                         if "mode" in action.attrs:
 353                                 mode = action.attrs["mode"]
 354                                 if (int(mode, 8) & 0222) == 0:
 355                                         engine.error(
 356                                         _("%(path)s has a preserve action, but is not writable (%(mode)s)") %  {"path": path, "mode": mode},
 357                                 msgid="%s%s.4" % (self.name, pkglint_id))
 358                         else:
 359                                 engine.error(
 360                                 _("%(path)s has a preserve action, but no mode") %  {"path": path, "mode": mode},
 361                                 msgid="%s%s.3" % (self.name, pkglint_id))
 362
 363                 # checks that require a physical file to look at
 364                 if self.proto_path is not None:
 365                         for directory in self.proto_path:
 366                                 fullpath = directory + "/" + path
 367
 368                                 if os.path.exists(fullpath):
 369                                         break
 370
 371                         if not os.path.exists(fullpath):
 372                                 engine.info(
 373                                         _("%s missing from proto area, skipping"
 374                                           " content checks") % path,
 375                                         msgid="%s%s.1" % (self.name, pkglint_id))
 376                         elif elf.is_elf_object(fullpath):
 377                                 # 32/64 bit in wrong place
 378                                 result = self.__elf_wrong_location_check(fullpath)
 379                                 if result != None:
 380                                         engine.error(result % path,
 381                                                 msgid="%s%s.2" % (self.name, pkglint_id))
 382                                 result = self.__elf_runpath_check(fullpath, engine)
 383                                 if result != None:
 384                                         engine.error(result % path,
 385                                                 msgid="%s%s.3" % (self.name, pkglint_id))
 386                                 # illumos does not support ASLR
 387                                 #result = self.__elf_aslr_check(fullpath, engine)
 388
 389         file_action.pkglint_desc = _("Paths should exist in the proto area.")
 390
 391         def link_resolves(self, action, manifest, engine, pkglint_id="002"):
 392                 """Checks for link resolution."""
 393
 394                 if action.name not in ["link", "hardlink"]:
 395                         return
 396
 397                 path = action.attrs["path"]
 398                 target = action.attrs["target"]
 399                 realtarget = self.__realpath(path, target)
 400
 401                 # Check against the target image (ref_paths), since links might
 402                 # resolve outside the packages delivering a particular
 403                 # component.
 404
 405                 # links to files should directly match a patch in the reference
 406                 # repo.
 407                 if self.ref_paths.get(realtarget, None):
 408                         return
 409
 410                 # If it didn't match a path in the reference repo, it may still
 411                 # be a link to a directory that has no action because it uses
 412                 # the default attributes.  Look for a path that starts with
 413                 # this value plus a trailing slash to be sure this it will be
 414                 # resolvable on a fully installed system.
 415                 realtarget += '/'
 416                 for key in self.ref_paths:
 417                         if key.startswith(realtarget):
 418                                 return
 419
 420                 engine.error(_("%s %s has unresolvable target '%s'") %
 421                                 (action.name, path, target),
 422                         msgid="%s%s.0" % (self.name, pkglint_id))
 423
 424         link_resolves.pkglint_desc = _("links should resolve.")
 425
 426         def init_script(self, action, manifest, engine, pkglint_id="003"):
 427                 """Checks for SVR4 startup scripts."""
 428
 429                 if action.name not in ["file", "dir", "link", "hardlink"]:
 430                         return
 431
 432                 path = action.attrs["path"]
 433                 if self.initscript_re.match(path):
 434                         engine.warning(
 435                                 _("SVR4 startup '%s', deliver SMF"
 436                                   " service instead") % path,
 437                                 msgid="%s%s.0" % (self.name, pkglint_id))
 438
 439         init_script.pkglint_desc = _(
 440                 "SVR4 startup scripts should not be delivered.")
 441
 442 class UserlandManifestChecker(base.ManifestChecker):
 443         """An opensolaris.org-specific class to check manifests."""
 444
 445         name = "userland.manifest"
 446
 447         def __init__(self, config):
 448                 super(UserlandManifestChecker, self).__init__(config)
 449
 450         def forbidden_publisher(self, manifest, engine, pkglint_id="1001"):
 451                 if not os.environ.get("ENCUMBERED"):
 452                         for action in manifest.gen_actions_by_type("depend"):
 453                                 for f in action.attrlist("fmri"):
 454                                         pkg_name=pkg.fmri.PkgFmri(f).pkg_name
 455                                         info_needed = pkg.client.api.PackageInfo.ALL_OPTIONS - \
 456                                             (pkg.client.api.PackageInfo.ACTION_OPTIONS |
 457                                              frozenset([pkg.client.api.PackageInfo.LICENSES]))
 458                                         progtracker = pkg.client.progress.NullProgressTracker()
 459                                         interface=pkg.client.api.ImageInterface("/", pkg.client.api.CURRENT_API_VERSION, progtracker, lambda x: False, None,None)
 460                                         ret = interface.info([pkg_name],True,info_needed)
 461                                         if ret[pkg.client.api.ImageInterface.INFO_FOUND]:
 462                                                 allowed_pubs = engine.get_param("%s.allowed_pubs" % self.name).split(" ") + ["unleashed","userland","userland-extra"]
 463                                                 for i in ret[pkg.client.api.ImageInterface.INFO_FOUND]:
 464                                                         if i.publisher not in allowed_pubs:
 465                                                                 engine.error(_("package %(pkg)s depends on %(name)s, which comes from forbidden publisher %(publisher)s") %
 466                                                                         {"pkg":manifest.fmri,"name":pkg_name,"publisher":i.publisher}, msgid="%s%s.1" % (self.name, pkglint_id))
 467
 468         forbidden_publisher.pkglint_desc = _(
 469                 "Dependencies should come from standard publishers" )
 470
 471         def component_check(self, manifest, engine, pkglint_id="001"):
 472                 manifest_paths = []
 473                 files = False
 474                 license = False
 475
 476                 for action in manifest.gen_actions_by_type("file"):
 477                         files = True
 478                         break
 479
 480                 if files == False:
 481                         return
 482
 483                 for action in manifest.gen_actions_by_type("license"):
 484                         if not action.attrs['license']:
 485                                 engine.error( _("missing vaue for action license attribute 'license' like 'CDDL','MIT','GPL'..."),
 486                                 msgid="%s%s.0" % (self.name, pkglint_id))
 487                         else:
 488                                 license = True
 489                                 break
 490
 491                 if license == False:
 492                         engine.error( _("missing license action"),
 493                                 msgid="%s%s.0" % (self.name, pkglint_id))
 494
 495 #               if 'org.opensolaris.arc-caseid' not in manifest:
 496 #                       engine.error( _("missing ARC data (org.opensolaris.arc-caseid)"),
 497 #                               msgid="%s%s.0" % (self.name, pkglint_id))
 498
 499         component_check.pkglint_desc = _(
 500                 "license actions and ARC information are required if you deliver files.")
 501
 502         def publisher_in_fmri(self, manifest, engine, pkglint_id="002"):
 503                 allowed_pubs = engine.get_param(
 504                     "%s.allowed_pubs" % self.name).split(" ")
 505
 506                 fmri = manifest.fmri
 507                 if fmri.publisher and fmri.publisher not in allowed_pubs:
 508                         engine.error(_("package %s has a publisher set!") %
 509                             manifest.fmri,
 510                             msgid="%s%s.2" % (self.name, pkglint_id))
 511
 512         publisher_in_fmri.pkglint_desc = _(
 513                 "extra publisher set" )