git-interface/git-update.py

   1 #!/usr/bin/python3
   2
   3 import os
   4 import pygit2
   5 import re
   6 import subprocess
   7 import sys
   8 import time
   9
  10 import srcinfo.parse
  11 import srcinfo.utils
  12
  13 import config
  14 import db
  15
  16 notify_cmd = config.get('notifications', 'notify-cmd')
  17
  18 repo_path = config.get('serve', 'repo-path')
  19 repo_regex = config.get('serve', 'repo-regex')
  20
  21 max_blob_size = config.getint('update', 'max-blob-size')
  22
  23
  24 def size_humanize(num):
  25     for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']:
  26         if abs(num) < 2048.0:
  27             if isinstance(num, int):
  28                 return "{}{}".format(num, unit)
  29             else:
  30                 return "{:.2f}{}".format(num, unit)
  31         num /= 1024.0
  32     return "{:.2f}{}".format(num, 'YiB')
  33
  34
  35 def extract_arch_fields(pkginfo, field):
  36     values = []
  37
  38     if field in pkginfo:
  39         for val in pkginfo[field]:
  40             values.append({"value": val, "arch": None})
  41
  42     for arch in ['i686', 'x86_64']:
  43         if field + '_' + arch in pkginfo:
  44             for val in pkginfo[field + '_' + arch]:
  45                 values.append({"value": val, "arch": arch})
  46
  47     return values
  48
  49
  50 def parse_dep(depstring):
  51     dep, _, desc = depstring.partition(': ')
  52     depname = re.sub(r'(<|=|>).*', '', dep)
  53     depcond = dep[len(depname):]
  54
  55     if (desc):
  56         return (depname + ': ' + desc, depcond)
  57     else:
  58         return (depname, depcond)
  59
  60
  61 def create_pkgbase(conn, pkgbase, user):
  62     cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
  63     userid = cur.fetchone()[0]
  64
  65     now = int(time.time())
  66     cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " +
  67                        "ModifiedTS, SubmitterUID, MaintainerUID) VALUES " +
  68                        "(?, ?, ?, ?, ?)", [pkgbase, now, now, userid, userid])
  69     pkgbase_id = cur.lastrowid
  70
  71     cur = conn.execute("INSERT INTO PackageNotifications " +
  72                        "(PackageBaseID, UserID) VALUES (?, ?)",
  73                        [pkgbase_id, userid])
  74
  75     conn.commit()
  76
  77     return pkgbase_id
  78
  79
  80 def save_metadata(metadata, conn, user):
  81     # Obtain package base ID and previous maintainer.
  82     pkgbase = metadata['pkgbase']
  83     cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases "
  84                        "WHERE Name = ?", [pkgbase])
  85     (pkgbase_id, maintainer_uid) = cur.fetchone()
  86     was_orphan = not maintainer_uid
  87
  88     # Obtain the user ID of the new maintainer.
  89     cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
  90     user_id = int(cur.fetchone()[0])
  91
  92     # Update package base details and delete current packages.
  93     now = int(time.time())
  94     conn.execute("UPDATE PackageBases SET ModifiedTS = ?, " +
  95                  "PackagerUID = ?, OutOfDateTS = NULL WHERE ID = ?",
  96                  [now, user_id, pkgbase_id])
  97     conn.execute("UPDATE PackageBases SET MaintainerUID = ? " +
  98                  "WHERE ID = ? AND MaintainerUID IS NULL",
  99                  [user_id, pkgbase_id])
 100     for table in ('Sources', 'Depends', 'Relations', 'Licenses', 'Groups'):
 101         conn.execute("DELETE FROM Package" + table + " WHERE EXISTS (" +
 102                      "SELECT * FROM Packages " +
 103                      "WHERE Packages.PackageBaseID = ? AND " +
 104                      "Package" + table + ".PackageID = Packages.ID)",
 105                      [pkgbase_id])
 106     conn.execute("DELETE FROM Packages WHERE PackageBaseID = ?", [pkgbase_id])
 107
 108     for pkgname in srcinfo.utils.get_package_names(metadata):
 109         pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 110
 111         if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
 112             ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
 113                                           pkginfo['pkgver'],
 114                                           pkginfo['pkgrel'])
 115         else:
 116             ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel'])
 117
 118         for field in ('pkgdesc', 'url'):
 119             if field not in pkginfo:
 120                 pkginfo[field] = None
 121
 122         # Create a new package.
 123         cur = conn.execute("INSERT INTO Packages (PackageBaseID, Name, " +
 124                            "Version, Description, URL) " +
 125                            "VALUES (?, ?, ?, ?, ?)",
 126                            [pkgbase_id, pkginfo['pkgname'], ver,
 127                             pkginfo['pkgdesc'], pkginfo['url']])
 128         conn.commit()
 129         pkgid = cur.lastrowid
 130
 131         # Add package sources.
 132         for source_info in extract_arch_fields(pkginfo, 'source'):
 133             conn.execute("INSERT INTO PackageSources (PackageID, Source, " +
 134                          "SourceArch) VALUES (?, ?, ?)",
 135                          [pkgid, source_info['value'], source_info['arch']])
 136
 137         # Add package dependencies.
 138         for deptype in ('depends', 'makedepends',
 139                         'checkdepends', 'optdepends'):
 140             cur = conn.execute("SELECT ID FROM DependencyTypes WHERE Name = ?",
 141                                [deptype])
 142             deptypeid = cur.fetchone()[0]
 143             for dep_info in extract_arch_fields(pkginfo, deptype):
 144                 depname, depcond = parse_dep(dep_info['value'])
 145                 deparch = dep_info['arch']
 146                 conn.execute("INSERT INTO PackageDepends (PackageID, " +
 147                              "DepTypeID, DepName, DepCondition, DepArch) " +
 148                              "VALUES (?, ?, ?, ?, ?)",
 149                              [pkgid, deptypeid, depname, depcond, deparch])
 150
 151         # Add package relations (conflicts, provides, replaces).
 152         for reltype in ('conflicts', 'provides', 'replaces'):
 153             cur = conn.execute("SELECT ID FROM RelationTypes WHERE Name = ?",
 154                                [reltype])
 155             reltypeid = cur.fetchone()[0]
 156             for rel_info in extract_arch_fields(pkginfo, reltype):
 157                 relname, relcond = parse_dep(rel_info['value'])
 158                 relarch = rel_info['arch']
 159                 conn.execute("INSERT INTO PackageRelations (PackageID, " +
 160                              "RelTypeID, RelName, RelCondition, RelArch) " +
 161                              "VALUES (?, ?, ?, ?, ?)",
 162                              [pkgid, reltypeid, relname, relcond, relarch])
 163
 164         # Add package licenses.
 165         if 'license' in pkginfo:
 166             for license in pkginfo['license']:
 167                 cur = conn.execute("SELECT ID FROM Licenses WHERE Name = ?",
 168                                    [license])
 169                 row = cur.fetchone()
 170                 if row:
 171                     licenseid = row[0]
 172                 else:
 173                     cur = conn.execute("INSERT INTO Licenses (Name) " +
 174                                        "VALUES (?)", [license])
 175                     conn.commit()
 176                     licenseid = cur.lastrowid
 177                 conn.execute("INSERT INTO PackageLicenses (PackageID, " +
 178                              "LicenseID) VALUES (?, ?)",
 179                              [pkgid, licenseid])
 180
 181         # Add package groups.
 182         if 'groups' in pkginfo:
 183             for group in pkginfo['groups']:
 184                 cur = conn.execute("SELECT ID FROM Groups WHERE Name = ?",
 185                                    [group])
 186                 row = cur.fetchone()
 187                 if row:
 188                     groupid = row[0]
 189                 else:
 190                     cur = conn.execute("INSERT INTO Groups (Name) VALUES (?)",
 191                                        [group])
 192                     conn.commit()
 193                     groupid = cur.lastrowid
 194                 conn.execute("INSERT INTO PackageGroups (PackageID, "
 195                              "GroupID) VALUES (?, ?)", [pkgid, groupid])
 196
 197     # Add user to notification list on adoption.
 198     if was_orphan:
 199         cur = conn.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " +
 200                            "PackageBaseID = ? AND UserID = ?",
 201                            [pkgbase_id, user_id])
 202         if cur.fetchone()[0] == 0:
 203             conn.execute("INSERT INTO PackageNotifications " +
 204                          "(PackageBaseID, UserID) VALUES (?, ?)",
 205                          [pkgbase_id, user_id])
 206
 207     conn.commit()
 208
 209
 210 def update_notify(conn, user, pkgbase_id):
 211     # Obtain the user ID of the new maintainer.
 212     cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user])
 213     user_id = int(cur.fetchone()[0])
 214
 215     # Execute the notification script.
 216     subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id)))
 217
 218
 219 def die(msg):
 220     sys.stderr.write("error: {:s}\n".format(msg))
 221     exit(1)
 222
 223
 224 def warn(msg):
 225     sys.stderr.write("warning: {:s}\n".format(msg))
 226
 227
 228 def die_commit(msg, commit):
 229     sys.stderr.write("error: The following error " +
 230                      "occurred when parsing commit\n")
 231     sys.stderr.write("error: {:s}:\n".format(commit))
 232     sys.stderr.write("error: {:s}\n".format(msg))
 233     exit(1)
 234
 235
 236 repo = pygit2.Repository(repo_path)
 237
 238 user = os.environ.get("AUR_USER")
 239 pkgbase = os.environ.get("AUR_PKGBASE")
 240 privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1')
 241 warn_or_die = warn if privileged else die
 242
 243 if len(sys.argv) == 2 and sys.argv[1] == "restore":
 244     if 'refs/heads/' + pkgbase not in repo.listall_references():
 245         die('{:s}: repository not found: {:s}'.format(sys.argv[1], pkgbase))
 246     refname = "refs/heads/master"
 247     sha1_old = sha1_new = repo.lookup_reference('refs/heads/' + pkgbase).target
 248 elif len(sys.argv) == 4:
 249     refname, sha1_old, sha1_new = sys.argv[1:4]
 250 else:
 251     die("invalid arguments")
 252
 253 if refname != "refs/heads/master":
 254     die("pushing to a branch other than master is restricted")
 255
 256 conn = db.Connection()
 257
 258 # Detect and deny non-fast-forwards.
 259 if sha1_old != "0000000000000000000000000000000000000000" and not privileged:
 260     walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL)
 261     walker.hide(sha1_new)
 262     if next(walker, None) is not None:
 263         die("denying non-fast-forward (you should pull first)")
 264
 265 # Prepare the walker that validates new commits.
 266 walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
 267 if sha1_old != "0000000000000000000000000000000000000000":
 268     walker.hide(sha1_old)
 269
 270 # Validate all new commits.
 271 for commit in walker:
 272     for fname in ('.SRCINFO', 'PKGBUILD'):
 273         if fname not in commit.tree:
 274             die_commit("missing {:s}".format(fname), str(commit.id))
 275
 276     for treeobj in commit.tree:
 277         blob = repo[treeobj.id]
 278
 279         if isinstance(blob, pygit2.Tree):
 280             die_commit("the repository must not contain subdirectories",
 281                        str(commit.id))
 282
 283         if not isinstance(blob, pygit2.Blob):
 284             die_commit("not a blob object: {:s}".format(treeobj),
 285                        str(commit.id))
 286
 287         if blob.size > max_blob_size:
 288             die_commit("maximum blob size ({:s}) exceeded".format(size_humanize(max_blob_size)), str(commit.id))
 289
 290     metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
 291     (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
 292     if errors:
 293         sys.stderr.write("error: The following errors occurred "
 294                          "when parsing .SRCINFO in commit\n")
 295         sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
 296         for error in errors:
 297             for err in error['error']:
 298                 sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err))
 299         exit(1)
 300
 301     metadata_pkgbase = metadata['pkgbase']
 302     if not re.match(repo_regex, metadata_pkgbase):
 303         die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
 304                    str(commit.id))
 305
 306     for pkgname in set(metadata['packages'].keys()):
 307         pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 308
 309         for field in ('pkgver', 'pkgrel', 'pkgname'):
 310             if field not in pkginfo:
 311                 die_commit('missing mandatory field: {:s}'.format(field),
 312                            str(commit.id))
 313
 314         if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit():
 315             die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']),
 316                        str(commit.id))
 317
 318         if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
 319             die_commit('invalid package name: {:s}'.format(pkginfo['pkgname']),
 320                        str(commit.id))
 321
 322         for field in ('pkgname', 'pkgdesc', 'url'):
 323             if field in pkginfo and len(pkginfo[field]) > 255:
 324                 die_commit('{:s} field too long: {:s}'.format(field, pkginfo[field]),
 325                            str(commit.id))
 326
 327         for field in ('install', 'changelog'):
 328             if field in pkginfo and not pkginfo[field] in commit.tree:
 329                 die_commit('missing {:s} file: {:s}'.format(field, pkginfo[field]),
 330                            str(commit.id))
 331
 332         for field in extract_arch_fields(pkginfo, 'source'):
 333             fname = field['value']
 334             if "://" in fname or "lp:" in fname:
 335                 continue
 336             if fname not in commit.tree:
 337                 die_commit('missing source file: {:s}'.format(fname),
 338                            str(commit.id))
 339
 340
 341 # Display a warning if .SRCINFO is unchanged.
 342 if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
 343     srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id
 344     srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id
 345     if srcinfo_id_old == srcinfo_id_new:
 346         warn(".SRCINFO unchanged. The package database will not be updated!")
 347
 348 # Read .SRCINFO from the HEAD commit.
 349 metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
 350 (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
 351
 352 # Ensure that the package base name matches the repository name.
 353 metadata_pkgbase = metadata['pkgbase']
 354 if metadata_pkgbase != pkgbase:
 355     die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase))
 356
 357 # Ensure that packages are neither blacklisted nor overwritten.
 358 pkgbase = metadata['pkgbase']
 359 cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase])
 360 row = cur.fetchone()
 361 pkgbase_id = row[0] if row else 0
 362
 363 cur = conn.execute("SELECT Name FROM PackageBlacklist")
 364 blacklist = [row[0] for row in cur.fetchall()]
 365
 366 cur = conn.execute("SELECT Name, Repo FROM OfficialProviders")
 367 providers = dict(cur.fetchall())
 368
 369 for pkgname in srcinfo.utils.get_package_names(metadata):
 370     pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 371     pkgname = pkginfo['pkgname']
 372
 373     if pkgname in blacklist:
 374         warn_or_die('package is blacklisted: {:s}'.format(pkgname))
 375     if pkgname in providers:
 376         repo = providers[pkgname]
 377         warn_or_die('package already provided by [{:s}]: {:s}'.format(repo, pkgname))
 378
 379     cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? AND " +
 380                        "PackageBaseID <> ?", [pkgname, pkgbase_id])
 381     if cur.fetchone()[0] > 0:
 382         die('cannot overwrite package: {:s}'.format(pkgname))
 383
 384 # Create a new package base if it does not exist yet.
 385 if pkgbase_id == 0:
 386     pkgbase_id = create_pkgbase(conn, pkgbase, user)
 387
 388 # Store package base details in the database.
 389 save_metadata(metadata, conn, user)
 390
 391 # Create (or update) a branch with the name of the package base for better
 392 # accessibility.
 393 repo.create_reference('refs/heads/' + pkgbase, sha1_new, True)
 394
 395 # Work around a Git bug: The HEAD ref is not updated when using gitnamespaces.
 396 # This can be removed once the bug fix is included in Git mainline. See
 397 # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html
 398 # for details.
 399 repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', sha1_new, True)
 400
 401 # Send package update notifications.
 402 update_notify(conn, user, pkgbase_id)
 403
 404 # Close the database.
 405 conn.close()