git-interface/git-update.py

   1 #!/usr/bin/python3
   2
   3 import configparser
   4 import mysql.connector
   5 import os
   6 import pygit2
   7 import re
   8 import subprocess
   9 import sys
  10
  11 import srcinfo.parse
  12 import srcinfo.utils
  13
  14 config = configparser.RawConfigParser()
  15 config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config")
  16
  17 aur_db_host = config.get('database', 'host')
  18 aur_db_name = config.get('database', 'name')
  19 aur_db_user = config.get('database', 'user')
  20 aur_db_pass = config.get('database', 'password')
  21 aur_db_socket = config.get('database', 'socket')
  22
  23 notify_cmd = config.get('notifications', 'notify-cmd')
  24
  25 repo_path = config.get('serve', 'repo-path')
  26 repo_regex = config.get('serve', 'repo-regex')
  27
  28
  29 def extract_arch_fields(pkginfo, field):
  30     values = []
  31
  32     if field in pkginfo:
  33         for val in pkginfo[field]:
  34             values.append({"value": val, "arch": None})
  35
  36     for arch in ['i686', 'x86_64']:
  37         if field + '_' + arch in pkginfo:
  38             for val in pkginfo[field + '_' + arch]:
  39                 values.append({"value": val, "arch": arch})
  40
  41     return values
  42
  43
  44 def parse_dep(depstring):
  45     dep, _, desc = depstring.partition(': ')
  46     depname = re.sub(r'(<|=|>).*', '', dep)
  47     depcond = dep[len(depname):]
  48
  49     if (desc):
  50         return (depname + ': ' + desc, depcond)
  51     else:
  52         return (depname, depcond)
  53
  54
  55 def save_metadata(metadata, db, cur, user):
  56     # Obtain package base ID and previous maintainer.
  57     pkgbase = metadata['pkgbase']
  58     cur.execute("SELECT ID, MaintainerUID FROM PackageBases "
  59                 "WHERE Name = %s", [pkgbase])
  60     (pkgbase_id, maintainer_uid) = cur.fetchone()
  61     was_orphan = not maintainer_uid
  62
  63     # Obtain the user ID of the new maintainer.
  64     cur.execute("SELECT ID FROM Users WHERE Username = %s", [user])
  65     user_id = int(cur.fetchone()[0])
  66
  67     # Update package base details and delete current packages.
  68     cur.execute("UPDATE PackageBases SET ModifiedTS = UNIX_TIMESTAMP(), " +
  69                 "PackagerUID = %s, OutOfDateTS = NULL WHERE ID = %s",
  70                 [user_id, pkgbase_id])
  71     cur.execute("UPDATE PackageBases SET MaintainerUID = %s " +
  72                 "WHERE ID = %s AND MaintainerUID IS NULL",
  73                 [user_id, pkgbase_id])
  74     cur.execute("DELETE FROM Packages WHERE PackageBaseID = %s",
  75                 [pkgbase_id])
  76
  77     for pkgname in srcinfo.utils.get_package_names(metadata):
  78         pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
  79
  80         if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0:
  81             ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']),
  82                                           pkginfo['pkgver'],
  83                                           pkginfo['pkgrel'])
  84         else:
  85             ver = '{:s}-{:s}'.format(pkginfo['pkgver'], pkginfo['pkgrel'])
  86
  87         for field in ('pkgdesc', 'url'):
  88             if field not in pkginfo:
  89                 pkginfo[field] = None
  90
  91         # Create a new package.
  92         cur.execute("INSERT INTO Packages (PackageBaseID, Name, " +
  93                     "Version, Description, URL) " +
  94                     "VALUES (%s, %s, %s, %s, %s)",
  95                     [pkgbase_id, pkginfo['pkgname'], ver,
  96                      pkginfo['pkgdesc'], pkginfo['url']])
  97         db.commit()
  98         pkgid = cur.lastrowid
  99
 100         # Add package sources.
 101         for source_info in extract_arch_fields(pkginfo, 'source'):
 102             cur.execute("INSERT INTO PackageSources (PackageID, Source, " +
 103                         "SourceArch) VALUES (%s, %s, %s)",
 104                         [pkgid, source_info['value'], source_info['arch']])
 105
 106         # Add package dependencies.
 107         for deptype in ('depends', 'makedepends',
 108                         'checkdepends', 'optdepends'):
 109             cur.execute("SELECT ID FROM DependencyTypes WHERE Name = %s",
 110                         [deptype])
 111             deptypeid = cur.fetchone()[0]
 112             for dep_info in extract_arch_fields(pkginfo, deptype):
 113                 depname, depcond = parse_dep(dep_info['value'])
 114                 deparch = dep_info['arch']
 115                 cur.execute("INSERT INTO PackageDepends (PackageID, " +
 116                             "DepTypeID, DepName, DepCondition, DepArch) " +
 117                             "VALUES (%s, %s, %s, %s, %s)",
 118                             [pkgid, deptypeid, depname, depcond, deparch])
 119
 120         # Add package relations (conflicts, provides, replaces).
 121         for reltype in ('conflicts', 'provides', 'replaces'):
 122             cur.execute("SELECT ID FROM RelationTypes WHERE Name = %s",
 123                         [reltype])
 124             reltypeid = cur.fetchone()[0]
 125             for rel_info in extract_arch_fields(pkginfo, reltype):
 126                 relname, relcond = parse_dep(rel_info['value'])
 127                 relarch = rel_info['arch']
 128                 cur.execute("INSERT INTO PackageRelations (PackageID, " +
 129                             "RelTypeID, RelName, RelCondition, RelArch) " +
 130                             "VALUES (%s, %s, %s, %s, %s)",
 131                             [pkgid, reltypeid, relname, relcond, relarch])
 132
 133         # Add package licenses.
 134         if 'license' in pkginfo:
 135             for license in pkginfo['license']:
 136                 cur.execute("SELECT ID FROM Licenses WHERE Name = %s",
 137                             [license])
 138                 if cur.rowcount == 1:
 139                     licenseid = cur.fetchone()[0]
 140                 else:
 141                     cur.execute("INSERT INTO Licenses (Name) VALUES (%s)",
 142                                 [license])
 143                     db.commit()
 144                     licenseid = cur.lastrowid
 145                 cur.execute("INSERT INTO PackageLicenses (PackageID, " +
 146                             "LicenseID) VALUES (%s, %s)",
 147                             [pkgid, licenseid])
 148
 149         # Add package groups.
 150         if 'groups' in pkginfo:
 151             for group in pkginfo['groups']:
 152                 cur.execute("SELECT ID FROM Groups WHERE Name = %s",
 153                             [group])
 154                 if cur.rowcount == 1:
 155                     groupid = cur.fetchone()[0]
 156                 else:
 157                     cur.execute("INSERT INTO Groups (Name) VALUES (%s)",
 158                                 [group])
 159                     db.commit()
 160                     groupid = cur.lastrowid
 161                 cur.execute("INSERT INTO PackageGroups (PackageID, "
 162                             "GroupID) VALUES (%s, %s)", [pkgid, groupid])
 163
 164     # Add user to notification list on adoption.
 165     if was_orphan:
 166         cur.execute("SELECT COUNT(*) FROM PackageNotifications WHERE " +
 167                     "PackageBaseID = %s AND UserID = %s",
 168                     [pkgbase_id, user_id])
 169         if cur.fetchone()[0] == 0:
 170             cur.execute("INSERT INTO PackageNotifications " +
 171                         "(PackageBaseID, UserID) VALUES (%s, %s)",
 172                         [pkgbase_id, user_id])
 173
 174     db.commit()
 175
 176
 177 def update_notify(db, cur, user, pkgbase_id):
 178     # Obtain the user ID of the new maintainer.
 179     cur.execute("SELECT ID FROM Users WHERE Username = %s", [user])
 180     user_id = int(cur.fetchone()[0])
 181
 182     # Execute the notification script.
 183     subprocess.Popen((notify_cmd, 'update', str(user_id), str(pkgbase_id)))
 184
 185
 186 def die(msg):
 187     sys.stderr.write("error: {:s}\n".format(msg))
 188     exit(1)
 189
 190
 191 def warn(msg):
 192     sys.stderr.write("warning: {:s}\n".format(msg))
 193
 194
 195 def die_commit(msg, commit):
 196     sys.stderr.write("error: The following error " +
 197                      "occurred when parsing commit\n")
 198     sys.stderr.write("error: {:s}:\n".format(commit))
 199     sys.stderr.write("error: {:s}\n".format(msg))
 200     exit(1)
 201
 202
 203 repo = pygit2.Repository(repo_path)
 204
 205 user = os.environ.get("AUR_USER")
 206 pkgbase = os.environ.get("AUR_PKGBASE")
 207 privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1')
 208 warn_or_die = warn if privileged else die
 209
 210 if len(sys.argv) == 2 and sys.argv[1] == "restore":
 211     if 'refs/heads/' + pkgbase not in repo.listall_references():
 212         die('{:s}: repository not found: {:s}'.format(sys.argv[1], pkgbase))
 213     refname = "refs/heads/master"
 214     sha1_old = sha1_new = repo.lookup_reference('refs/heads/' + pkgbase).target
 215 elif len(sys.argv) == 4:
 216     refname, sha1_old, sha1_new = sys.argv[1:4]
 217 else:
 218     die("invalid arguments")
 219
 220 if refname != "refs/heads/master":
 221     die("pushing to a branch other than master is restricted")
 222
 223 db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
 224                              passwd=aur_db_pass, db=aur_db_name,
 225                              unix_socket=aur_db_socket, buffered=True)
 226 cur = db.cursor()
 227
 228 # Detect and deny non-fast-forwards.
 229 if sha1_old != "0000000000000000000000000000000000000000" and not privileged:
 230     walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL)
 231     walker.hide(sha1_new)
 232     if next(walker, None) is not None:
 233         die("denying non-fast-forward (you should pull first)")
 234
 235 # Prepare the walker that validates new commits.
 236 walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
 237 if sha1_old != "0000000000000000000000000000000000000000":
 238     walker.hide(sha1_old)
 239
 240 # Validate all new commits.
 241 for commit in walker:
 242     for fname in ('.SRCINFO', 'PKGBUILD'):
 243         if fname not in commit.tree:
 244             die_commit("missing {:s}".format(fname), str(commit.id))
 245
 246     for treeobj in commit.tree:
 247         blob = repo[treeobj.id]
 248
 249         if isinstance(blob, pygit2.Tree):
 250             die_commit("the repository must not contain subdirectories",
 251                        str(commit.id))
 252
 253         if not isinstance(blob, pygit2.Blob):
 254             die_commit("not a blob object: {:s}".format(treeobj),
 255                        str(commit.id))
 256
 257         if blob.size > 250000:
 258             die_commit("maximum blob size (250kB) exceeded", str(commit.id))
 259
 260     metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode()
 261     (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
 262     if errors:
 263         sys.stderr.write("error: The following errors occurred "
 264                          "when parsing .SRCINFO in commit\n")
 265         sys.stderr.write("error: {:s}:\n".format(str(commit.id)))
 266         for error in errors:
 267             for err in error['error']:
 268                 sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err))
 269         exit(1)
 270
 271     metadata_pkgbase = metadata['pkgbase']
 272     if not re.match(repo_regex, metadata_pkgbase):
 273         die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase),
 274                    str(commit.id))
 275
 276     for pkgname in set(metadata['packages'].keys()):
 277         pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 278
 279         for field in ('pkgver', 'pkgrel', 'pkgname'):
 280             if field not in pkginfo:
 281                 die_commit('missing mandatory field: {:s}'.format(field),
 282                            str(commit.id))
 283
 284         if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit():
 285             die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']),
 286                        str(commit.id))
 287
 288         if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
 289             die_commit('invalid package name: {:s}'.format(pkginfo['pkgname']),
 290                        str(commit.id))
 291
 292         for field in ('pkgname', 'pkgdesc', 'url'):
 293             if field in pkginfo and len(pkginfo[field]) > 255:
 294                 die_commit('{:s} field too long: {:s}'.format(field, pkginfo[field]),
 295                            str(commit.id))
 296
 297         for field in ('install', 'changelog'):
 298             if field in pkginfo and not pkginfo[field] in commit.tree:
 299                 die_commit('missing {:s} file: {:s}'.format(field, pkginfo[field]),
 300                            str(commit.id))
 301
 302         for field in extract_arch_fields(pkginfo, 'source'):
 303             fname = field['value']
 304             if "://" in fname or "lp:" in fname:
 305                 continue
 306             if fname not in commit.tree:
 307                 die_commit('missing source file: {:s}'.format(fname),
 308                            str(commit.id))
 309
 310
 311 # Display a warning if .SRCINFO is unchanged.
 312 if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new):
 313     srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id
 314     srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id
 315     if srcinfo_id_old == srcinfo_id_new:
 316         warn(".SRCINFO unchanged. The package database will not be updated!")
 317
 318 # Read .SRCINFO from the HEAD commit.
 319 metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode()
 320 (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw)
 321
 322 # Ensure that the package base name matches the repository name.
 323 metadata_pkgbase = metadata['pkgbase']
 324 if metadata_pkgbase != pkgbase:
 325     die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase))
 326
 327 # Ensure that packages are neither blacklisted nor overwritten.
 328 pkgbase = metadata['pkgbase']
 329 cur.execute("SELECT ID FROM PackageBases WHERE Name = %s", [pkgbase])
 330 pkgbase_id = cur.fetchone()[0] if cur.rowcount == 1 else 0
 331
 332 cur.execute("SELECT Name FROM PackageBlacklist")
 333 blacklist = [row[0] for row in cur.fetchall()]
 334
 335 cur.execute("SELECT Name, Repo FROM OfficialProviders")
 336 providers = dict(cur.fetchall())
 337
 338 for pkgname in srcinfo.utils.get_package_names(metadata):
 339     pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata)
 340     pkgname = pkginfo['pkgname']
 341
 342     if pkgname in blacklist:
 343         warn_or_die('package is blacklisted: {:s}'.format(pkgname))
 344     if pkgname in providers:
 345         repo = providers[pkgname]
 346         warn_or_die('package already provided by [{:s}]: {:s}'.format(repo, pkgname))
 347
 348     cur.execute("SELECT COUNT(*) FROM Packages WHERE Name = %s AND " +
 349                 "PackageBaseID <> %s", [pkgname, pkgbase_id])
 350     if cur.fetchone()[0] > 0:
 351         die('cannot overwrite package: {:s}'.format(pkgname))
 352
 353 # Store package base details in the database.
 354 save_metadata(metadata, db, cur, user)
 355
 356 # Create (or update) a branch with the name of the package base for better
 357 # accessibility.
 358 repo.create_reference('refs/heads/' + pkgbase, sha1_new, True)
 359
 360 # Work around a Git bug: The HEAD ref is not updated when using gitnamespaces.
 361 # This can be removed once the bug fix is included in Git mainline. See
 362 # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref-namespace-td7632149.html
 363 # for details.
 364 repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', sha1_new, True)
 365
 366 # Send package update notifications.
 367 update_notify(db, cur, user, pkgbase_id)
 368
 369 # Close the database.
 370 db.close()