bin/rm-config

   1 #! /usr/bin/python3
   2
   3 import argparse
   4 import hashlib
   5 import io
   6 import logging
   7 from pathlib import Path
   8 import re
   9 import shlex
  10 import subprocess
  11 import sys
  12 import tempfile
  13 from xdg.BaseDirectory import xdg_config_home  # type: ignore
  14 from voluptuous import Any, Schema  # type: ignore
  15 from voluptuous.validators import (  # type: ignore
  16     And, Date, IsDir, IsFile, Match, NotIn
  17 )
  18 import yaml
  19
  20 LOG_FORMAT = "%(levelname)s %(message)s"
  21 log = logging.getLogger()
  22
  23 STAGES = [
  24     "base",
  25     "built-almost-final",
  26     "finalized-changelog",
  27     "reproduced-images",
  28     "built-iuks",
  29 ]
  30
  31 # pylint: disable=E1120
  32 InputStr = And(str, NotIn(["FIXME"]))
  33 IsBuildManifest = And(IsFile(), Match(re.compile(r".*[.]build-manifest$")))
  34 IsIsoFile = And(IsFile(), Match(re.compile(r".*[.]iso$")))
  35 IsImgFile = And(IsFile(), Match(re.compile(r".*[.]img$")))
  36
  37 STAGE_SCHEMA = {
  38     "base": {
  39         "tails_signature_key": InputStr,
  40         "isos": IsDir(),
  41         "artifacts": IsDir(),
  42         "master_checkout": IsDir(),
  43         "release_checkout": IsDir(),
  44         "version": InputStr,
  45         "previous_version": InputStr,
  46         "previous_stable_version": InputStr,
  47         "next_planned_major_version": InputStr,
  48         "second_next_planned_major_version": InputStr,
  49         "next_planned_bugfix_version": InputStr,
  50         "next_planned_version": InputStr,
  51         "next_potential_emergency_version": InputStr,
  52         "next_stable_changelog_version": InputStr,
  53         "release_date": Date(),
  54         "major_release": Any(0, 1),
  55         "dist": Any("stable", "alpha"),
  56         "release_branch": InputStr,
  57         "tag": InputStr,
  58         "previous_tag": InputStr,
  59         "website_release_branch": InputStr,
  60         "iuks_dir": IsDir(),
  61         "iuks_hashes": InputStr,
  62         "milestone": InputStr,
  63         "tails_signature_key_long_id": InputStr,
  64         "iuk_source_versions": InputStr,
  65         "test_iuk_source_versions": InputStr,
  66     },
  67     "built-almost-final": {
  68         "almost_final_build_manifest": IsBuildManifest,
  69     },
  70     "finalized-changelog": {
  71         "source_date_epoch": int,
  72     },
  73     "reproduced-images": {
  74         "matching_jenkins_images_build_id": int,
  75     },
  76     "built-iuks": {
  77         "iso_path": IsIsoFile,
  78         "img_path": IsImgFile,
  79         "iso_sha256sum": str,
  80         "img_sha256sum": str,
  81         "iso_size_in_bytes": int,
  82         "img_size_in_bytes": int,
  83         "candidate_jenkins_iuks_build_id": int,
  84         "iuks_hashes": IsFile(),
  85     }
  86 }
  87 # pylint: enable=E1120
  88
  89
  90 def git_repo_root():
  91     """Returns the root of the current Git repository as a Path object"""
  92     return Path(
  93         subprocess.check_output(["git", "rev-parse", "--show-toplevel"],
  94                                 encoding="utf8").rstrip("\n"))
  95
  96
  97 def sha256_file(filename):
  98     """Returns the hex-encoded SHA256 hash of FILENAME"""
  99     sha256 = hashlib.sha256()
 100     with io.open(filename, mode="rb") as input_fd:
 101         content = input_fd.read()
 102         sha256.update(content)
 103     return sha256.hexdigest()
 104
 105
 106 class Config():
 107     """Load, validate, generate, and output Release Management configuration"""
 108     def __init__(self, stage: str):
 109         self.stage = stage
 110         self.config_files = [
 111             git_repo_root() / "config/release_management/defaults.yml"
 112         ] + list(
 113             (Path(xdg_config_home) / "tails/release_management").glob("*.yml"))
 114         self.data = self.load_config_files()
 115         self.data.update(self.generate_config())
 116         log.debug("Configuration:\n%s", self.data)
 117         self.validate()
 118
 119     def load_config_files(self):
 120         """
 121         Load all relevant configuration files and return the resulting
 122         configuration dict
 123         """
 124         data = {}
 125         for config_file in self.config_files:
 126             log.debug("Loading %s", config_file)
 127             data.update(yaml.safe_load(open(config_file, 'r')))
 128         return data
 129
 130     def generate_config(self):
 131         """
 132         Returns a dict of supplemental, programmatically-generated,
 133         configuration.
 134         """
 135         version = self.data["version"]
 136         tails_signature_key = self.data["tails_signature_key"]
 137         tag = version.replace("~", "-")
 138         release_branch = "testing" \
 139             if self.data["major_release"] == 1 \
 140             else "stable"
 141         iuks_dir = Path(self.data["isos"]) / "iuks/v2"
 142         iuk_hashes = Path(iuks_dir) / ("to_%s.sha256sum" % version)
 143         iuk_source_versions = subprocess.check_output(
 144             [git_repo_root() / "bin/iuk-source-versions", version],
 145             encoding="utf8").rstrip("\n")
 146         # We always test the upgrade path from the last stable version
 147         test_iuk_source_versions = self.data['previous_stable_version']
 148         # ... but also from the last of any alphas/betas/RCs. Given Tails
 149         # versioning scheme we trivially have that it can only be the
 150         # previous version, but only if it isn't the same as the previous
 151         # stable release (alpha/beta/RC is not stable by definition).
 152         if self.data['previous_stable_version'] != self.data['previous_version']:
 153             test_iuk_source_versions += ' ' + self.data['previous_version']
 154         generated_config = {
 155             "release_branch": release_branch,
 156             "tag": tag,
 157             "previous_tag": self.data["previous_version"].replace("~", "-"),
 158             "website_release_branch": "web/release-%s" % tag,
 159             "iuk_source_versions": iuk_source_versions,
 160             "test_iuk_source_versions": test_iuk_source_versions,
 161             "iuks_dir": str(iuks_dir),
 162             "iuks_hashes": str(iuk_hashes),
 163             "milestone": re.sub('~.*', '', self.data["version"]),
 164             "tails_signature_key_long_id": tails_signature_key[24:],
 165         }
 166         if self.stage == 'built-iuks':
 167             iso_path = Path(self.data["isos"]) \
 168                 / ("tails-amd64-%s/tails-amd64-%s.iso" % (version, version))
 169             img_path = Path(self.data["isos"]) \
 170                 / ("tails-amd64-%s/tails-amd64-%s.img" % (version, version))
 171             generated_config.update({
 172                 "iso_path": str(iso_path),
 173                 "img_path": str(img_path),
 174                 "iso_sha256sum": sha256_file(iso_path),
 175                 "img_sha256sum": sha256_file(img_path),
 176                 "iso_size_in_bytes": iso_path.stat().st_size,
 177                 "img_size_in_bytes": img_path.stat().st_size,
 178             })
 179         return generated_config
 180
 181     def schema(self):
 182         """
 183         Returns a configuration validation schema function for
 184         the current stage
 185         """
 186         schema = {}
 187         for stage in STAGES:
 188             schema.update(STAGE_SCHEMA[stage])
 189             if stage == self.stage:
 190                 break
 191         log.debug("Schema:\n%s", schema)
 192         return Schema(schema, required=True)
 193
 194     def validate(self):
 195         """Checks that the configuration is valid, else raise exception"""
 196         schema = self.schema()
 197         schema(self.data)
 198
 199     def to_shell(self):
 200         """
 201         Returns shell commands that, if executed, would export the
 202         configuration into the environment.
 203         """
 204         return "\n".join([
 205             "export %(key)s=%(val)s" % {
 206                 "key": k.upper(),
 207                 "val": shlex.quote(str(v))
 208             } for (k, v) in self.data.items()
 209         ]) + "\n"
 210
 211
 212 def generate_boilerplate(stage: str):
 213     """Generate boilerplate for STAGE"""
 214     log.debug("Generating boilerplate for stage '%s'", stage)
 215     with open(git_repo_root() /
 216               ("config/release_management/templates/%s.yml" % stage)) as src:
 217         with open(
 218                 Path(xdg_config_home) / "tails/release_management/current.yml",
 219                 'a') as dst:
 220             dst.write(src.read())
 221
 222
 223 def generate_environment(stage: str):
 224     """
 225     Prints to stdout the path to a file that contains commands
 226     that export the configuration for STAGE to the environment.
 227     """
 228     log.debug("Generating environment for stage '%s'", stage)
 229     config = Config(stage=stage)
 230     shell_snippet = tempfile.NamedTemporaryFile(delete=False)
 231     with open(shell_snippet.name, 'w') as shell_snippet_fd:
 232         shell_snippet_fd.write(config.to_shell())
 233     print(shell_snippet.name)
 234
 235
 236 def validate_configuration(stage: str):
 237     """Validate configuration for STAGE, raise exception if invalid"""
 238     log.debug("Validating configuration for stage '%s'", stage)
 239     Config(stage=stage)
 240     log.info("Configuration is valid")
 241
 242
 243 def main():
 244     """Command-line entry point"""
 245     parser = argparse.ArgumentParser(
 246         description="Query and manage Release Management configuration",
 247         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 248         )
 249     parser.add_argument("--debug", action="store_true", help="debug output")
 250     subparsers = parser.add_subparsers(help="sub-command help", dest="command")
 251
 252     parser_generate_boilerplate = subparsers.add_parser(
 253         "generate-boilerplate",
 254         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 255         help="Creates a configuration file template that you will fill")
 256     parser_generate_boilerplate.add_argument("--stage",
 257                                              type=str,
 258                                              action="store",
 259                                              default="base",
 260                                              choices=STAGES,
 261                                              help="Select stage")
 262     parser_generate_boilerplate.set_defaults(func=generate_boilerplate)
 263
 264     parser_validate_configuration = subparsers.add_parser(
 265         "validate-configuration",
 266         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 267         help="Validate configuration files")
 268     parser_validate_configuration.add_argument("--stage",
 269                                                type=str,
 270                                                action="store",
 271                                                default="base",
 272                                                choices=STAGES,
 273                                                help="Select stage")
 274     parser_validate_configuration.set_defaults(func=validate_configuration)
 275
 276     parser_generate_environment = subparsers.add_parser(
 277         "generate-environment",
 278         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 279         help="Creates a shell sourceable file with resulting environment")
 280     parser_generate_environment.add_argument("--stage",
 281                                              type=str,
 282                                              action="store",
 283                                              default="base",
 284                                              choices=STAGES,
 285                                              help="Select stage")
 286     parser_generate_environment.set_defaults(func=generate_environment)
 287
 288     args = parser.parse_args()
 289
 290     if args.debug:
 291         logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
 292     else:
 293         logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
 294
 295     if args.command is None:
 296         parser.print_help()
 297     else:
 298         args.func(stage=args.stage)
 299
 300
 301 if __name__ == '__main__':
 302     sys.exit(main())