git-interface/git-update.py

   1 #!/usr/bin/python3
   2
   3 import os
   4 import pygit2
   5 import re
   6 import subprocess
   7 import sys
   8 import time
   9
  10 import srcinfo.parse
  11 import srcinfo.utils
  12
  13 import config
  14 import db
  15
  16 notify_cmd = config.get('notifications', 'notify-cmd')
  17
  18 repo_path = config.get('serve', 'repo-path')
  19 repo_regex = config.get('serve', 'repo-regex')
  20
  21 max_blob_size = config.getint('update', 'max-blob-size')
  22
  23
  24 def size_humanize(num):
  25     for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']:
  26         if abs(num) < 2048.0:
  27             if isinstance(num, int):
  28                 return "{}{}".format(num, unit)
  29             else:
  30                 return "{:.2f}{}".format(num, unit)
  31         num /= 1024.0
  32     return "{:.2f}{}".format(num, 'YiB')
  33
  34
  35 def extract_arch_fields(pkginfo, field):
  36     values = []
  37
  38     if field in pkginfo:
  39         for val in pkginfo[field]:
  40             values.append({"value": val, "arch": None})
  41
  42     for arch in ['i686', 'x86_64']:
  43         if field + '_' + arch in pkginfo:
  44             for val in pkginfo[field + '_' + arch]:
  45                 values.append({"value": val, "arch": arch})
  46
  47     return values
  48
  49
  50 def parse_dep(depstring):
  51     dep, _, desc = depstring.partition(': ')
  52     depname = re.sub(r'(<|=|>).*', '', dep)
  53     depcond = dep[len(depname):]
  54
  55     if (desc):
  56         return (depname + ': ' + desc, depcond)
  57     else:
  58         return (depname, depcond)
  59
  60
  61 def save_metadata(metadata, conn, user):
  62     # Obtain package base ID and previous maintainer.
  63     pkgbase = metadata['pkgbase']
  64     cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases "
  65                        "WHERE Name = ?", [pkgbase])
  66     (pkgbase_id, maintainer_uid) = cur.fetchone()
  67     was_orphan = not maintainer_uid
  68
  69     # Obtain the user ID of the new maintainer.
  70     cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
  71     user_id = int(cur.fetchone()[0])
  72
  73     # Update package base details and delete current packages.
  74     now = int(time.time())
  75     conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " +
  76                  "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?",
  77                  [now, user_id, pkgbase_id])
  78     conn.execute("UPDATE PackageBases SET MaintainerUID = ? " +
  79                  "WHERE ID = ? AND MaintainerUID IS NULL",
  80                  [user_id, pkgbase_id])
  81     for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'):
  82         conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" +
  83                      "SELECT * FROM Packages " +
  84                      "WHERE Packages.PackageBaseID = ? AND " +
  85                      "Package" + table + ".PackageID = Packages.ID)",
  86                      [pkgbase_id])
  87     conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id])
  88
  89     for pkgname in srcinfo.utils.get_package_names(metadata):
  90         pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
  91
  92         if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
  93             ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
  94                                           pkginfo['pkgver'],
  95                                           pkginfo['pkgrel'])
  96         else:
  97             ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel'])
  98
  99         for field in ('pkgdesc', 'url'):
 100             if field not in pkginfo:
 101                 pkginfo[field] = None
 102
 103         # Create a new package.
 104         cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " +
 105                            "Version, Description, URL) " +
 106                            "VALUES (?, ?, ?, ?, ?)",
 107                            [pkgbase_id, pkginfo['pkgname'], ver,
 108                             pkginfo['pkgdesc'], pkginfo['url']])
 109         conn.commit()
 110         pkgid = cur.lastrowid
 111
 112         # Add package sources.
 113         for source_info in extract_arch_fields(pkginfo, 'source'):
 114             conn.execute("INSERT INTO PackageSources (PackageID, Source, " +
 115                          "SourceArch) VALUES (?, ?, ?)",
 116                          [pkgid, source_info['value'], source_info['arch']])
 117
 118         # Add package dependencies.
 119         for deptype in ('depends', 'makedepends',
 120                         'checkdepends', 'optdepends'):
 121             cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?",
 122                                [deptype])
 123             deptypeid = cur.fetchone()[0]
 124             for dep_info in extract_arch_fields(pkginfo, deptype):
 125                 depname, depcond = parse_dep(dep_info['value'])
 126                 deparch = dep_info['arch']
 127                 conn.execute("INSERT INTO PackageDepends (PackageID, " +
 128                              "DepTypeID, DepName, DepCondition, DepArch) " +
 129                              "VALUES (?, ?, ?, ?, ?)",
 130                              [pkgid, deptypeid, depname, depcond, deparch])
 131
 132         # Add package relations (conflicts, provides, replaces).
 133         for reltype in ('conflicts', 'provides', 'replaces'):
 134             cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?",
 135                                [reltype])
 136             reltypeid = cur.fetchone()[0]
 137             for rel_info in extract_arch_fields(pkginfo, reltype):
 138                 relname, relcond = parse_dep(rel_info['value'])
 139                 relarch = rel_info['arch']
 140                 conn.execute("INSERT INTO PackageRelations (PackageID, " +
 141                              "RelTypeID, RelName, RelCondition, RelArch) " +
 142                              "VALUES (?, ?, ?, ?, ?)",
 143                              [pkgid, reltypeid, relname, relcond, relarch])
 144
 145         # Add package licenses.
 146         if 'license' in pkginfo:
 147             for license in pkginfo['license']:
 148                 cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?",
 149                                    [license])
 150                 row = cur.fetchone()
 151                 if row:
 152                     licenseid = row[0]
 153                 else:
 154                     cur = conn.execute("INSERT INTO Licenses (Name) " +
 155                                        "VALUES (?)", [license])
 156                     conn.commit()
 157                     licenseid = cur.lastrowid
 158                 conn.execute("INSERT INTO PackageLicenses (PackageID, " +
 159                              "LicenseID) VALUES (?, ?)",
 160                              [pkgid, licenseid])
 161
 162         # Add package groups.
 163         if 'groups' in pkginfo:
 164             for group in pkginfo['groups']:
 165                 cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?",
 166                                    [group])
 167                 row = cur.fetchone()
 168                 if row:
 169                     groupid = row[0]
 170                 else:
 171                     cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)",
 172                                        [group])
 173                     conn.commit()
 174                     groupid = cur.lastrowid
 175                 conn.execute("INSERT INTO PackageGroups (PackageID, "
 176                              "GroupID) VALUES (?, ?)", [pkgid, groupid])
 177
 178     # Add user to notification list on adoption.
 179     if was_orphan:
 180         cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " +
 181                            "PackageBaseID = ? AND UserID = ?",
 182                            [pkgbase_id, user_id])
 183         if cur.fetchone()[0] == 0:
 184             conn.execute("INSERT INTO PackageNotifications " +
 185                          "(PackageBaseID, UserID) VALUES (?, ?)",
 186                          [pkgbase_id, user_id])
 187
 188     conn.commit()
 189
 190
 191 def update_notify(conn, user, pkgbase_id):
 192     # Obtain the user ID of the new maintainer.
 193     cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
 194     user_id = int(cur.fetchone()[0])
 195
 196     # Execute the notification script.
 197     subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id)))
 198
 199
 200 def die(msg):
 201     sys.stderr.write("error: {:s}\n".format(msg))
 202     exit(1)
 203
 204
 205 def warn(msg):
 206     sys.stderr.write("warning: {:s}\n".format(msg))
 207
 208
 209 def die_commit(msg, commit):
 210     sys.stderr.write("error: The following error " +
 211                      "occurred when parsing commit\n")
 212     sys.stderr.write("error: {:s}:\n".format(commit))
 213     sys.stderr.write("error: {:s}\n".format(msg))
 214     exit(1)
 215
 216
 217 repo = pygit2.Repository(repo_path)
 218
 219 user = os.environ.get("AUR_USER")
 220 pkgbase = os.environ.get("AUR_PKGBASE")
 221 privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1')
 222 warn_or_die = warn if privileged else die
 223
 224 if len(sys.argv) == 2 and sys.argv[1] == "restore":
 225     if 'refs/heads/' + pkgbase not in repo.listall_references():
 226         die('{:s}: repository not found: {:s}'.format(sys.argv[1], pkgbase))
 227     refname = "refs/heads/master"
 228     sha1_old = sha1_new = repo.lookup_reference('refs/heads/' + pkgbase).target
 229 elif len(sys.argv) == 4:
 230     refname, sha1_old, sha1_new = sys.argv[1:4]
 231 else:
 232     die("invalid arguments")
 233
 234 if refname != "refs/heads/master":
 235     die("pushing to a branch other than master is restricted")
 236
 237 conn = db.Connection()
 238
 239 # Detect and deny non-fast-forwards.
 240 if sha1_old != "0000000000000000000000000000000000000000" and not privileged:
 241     walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL)
 242     walker.hide(sha1_new)
 243     if next(walker, None) is not None:
 244         die("denying non-fast-forward (you should pull first)")
 245
 246 # Prepare the walker that validates new commits.
 247 walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
 248 if sha1_old != "0000000000000000000000000000000000000000":
 249     walker.hide(sha1_old)
 250
 251 # Validate all new commits.
 252 for commit in walker:
 253     for fname in ('.SRCINFO', 'PKGBUILD'):
 254         if fname not in commit.tree:
 255             die_commit("missing {:s}".format(fname), str(commit.id))
 256
 257     for treeobj in commit.tree:
 258         blob = repo[treeobj.id]
 259
 260         if isinstance(blob, pygit2.Tree):
 261             die_commit("the repository must not contain subdirectories",
 262                        str(commit.id))
 263
 264         if not isinstance(blob, pygit2.Blob):
 265             die_commit("not a blob object: {:s}".format(treeobj),
 266                        str(commit.id))
 267
 268         if blob.size > max_blob_size:
 269             die_commit("maximum blob size ({:s}) exceeded".format(size_humanize(max_blob_size)), str(commit.id))
 270
 271     metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
 272     (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
 273     if errors:
 274         sys.stderr.write("error: The following errors occurred "
 275                          "when parsing .SRCINFO in commit\n")
 276         sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
 277         for error in errors:
 278             for err in error['error']:
 279                 sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err))
 280         exit(1)
 281
 282     metadata_pkgbase = metadata['pkgbase']
 283     if not re.match(repo_regex, metadata_pkgbase):
 284         die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
 285                    str(commit.id))
 286
 287     for pkgname in set(metadata['packages'].keys()):
 288         pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 289
 290         for field in ('pkgver', 'pkgrel', 'pkgname'):
 291             if field not in pkginfo:
 292                 die_commit('missing mandatory field: {:s}'.format(field),
 293                            str(commit.id))
 294
 295         if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit():
 296             die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']),
 297                        str(commit.id))
 298
 299         if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
 300             die_commit('invalid package name: {:s}'.format(pkginfo['pkgname']),
 301                        str(commit.id))
 302
 303         for field in ('pkgname', 'pkgdesc', 'url'):
 304             if field in pkginfo and len(pkginfo[field]) > 255:
 305                 die_commit('{:s} field too long: {:s}'.format(field, pkginfo[field]),
 306                            str(commit.id))
 307
 308         for field in ('install', 'changelog'):
 309             if field in pkginfo and not pkginfo[field] in commit.tree:
 310                 die_commit('missing {:s} file: {:s}'.format(field, pkginfo[field]),
 311                            str(commit.id))
 312
 313         for field in extract_arch_fields(pkginfo, 'source'):
 314             fname = field['value']
 315             if "://" in fname or "lp:" in fname:
 316                 continue
 317             if fname not in commit.tree:
 318                 die_commit('missing source file: {:s}'.format(fname),
 319                            str(commit.id))
 320
 321
 322 # Display a warning if .SRCINFO is unchanged.
 323 if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
 324     srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id
 325     srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id
 326     if srcinfo_id_old == srcinfo_id_new:
 327         warn(".SRCINFO unchanged. The package database will not be updated!")
 328
 329 # Read .SRCINFO from the HEAD commit.
 330 metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
 331 (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
 332
 333 # Ensure that the package base name matches the repository name.
 334 metadata_pkgbase = metadata['pkgbase']
 335 if metadata_pkgbase != pkgbase:
 336     die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase))
 337
 338 # Ensure that packages are neither blacklisted nor overwritten.
 339 pkgbase = metadata['pkgbase']
 340 cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase])
 341 row = cur.fetchone()
 342 pkgbase_id = row[0] if row else 0
 343
 344 cur = conn.execute("SELECT Name FROM PackageBlacklist")
 345 blacklist = [row[0] for row in cur.fetchall()]
 346
 347 cur = conn.execute("SELECT Name, Repo FROM OfficialProviders")
 348 providers = dict(cur.fetchall())
 349
 350 for pkgname in srcinfo.utils.get_package_names(metadata):
 351     pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 352     pkgname = pkginfo['pkgname']
 353
 354     if pkgname in blacklist:
 355         warn_or_die('package is blacklisted: {:s}'.format(pkgname))
 356     if pkgname in providers:
 357         repo = providers[pkgname]
 358         warn_or_die('package already provided by [{:s}]: {:s}'.format(repo, pkgname))
 359
 360     cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? AND " +
 361                        "PackageBaseID <> ?", [pkgname, pkgbase_id])
 362     if cur.fetchone()[0] > 0:
 363         die('cannot overwrite package: {:s}'.format(pkgname))
 364
 365 # Store package base details in the database.
 366 save_metadata(metadata, conn, user)
 367
 368 # Create (or update) a branch with the name of the package base for better
 369 # accessibility.
 370 repo.create_reference('refs/heads/' + pkgbase, sha1_new, True)
 371
 372 # Work around a Git bug: The HEAD ref is not updated when using gitnamespaces.
 373 # This can be removed once the bug fix is included in Git mainline. See
 374 # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html
 375 # for details.
 376 repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', sha1_new, True)
 377
 378 # Send package update notifications.
 379 update_notify(conn, user, pkgbase_id)
 380
 381 # Close the database.
 382 conn.close()