security/manager/tools/getCTKnownLogs.py

   1 #!/usr/bin/env python
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 """
   7 Parses a JSON file listing the known Certificate Transparency logs
   8 (log_list.json) and generates a C++ header file to be included in Firefox.
   9
  10 The current log_list.json file available under security/manager/tools
  11 was originally downloaded from
  12 https://www.certificate-transparency.org/known-logs
  13 and edited to include the disqualification time for the disqualified logs using
  14 https://cs.chromium.org/chromium/src/net/cert/ct_known_logs_static-inc.h
  15 """
  16
  17 import argparse
  18 import base64
  19 import datetime
  20 import json
  21 import os.path
  22 import sys
  23 import textwrap
  24 from string import Template
  25
  26 import six
  27 import urllib3
  28
  29
  30 def decodebytes(s):
  31     if six.PY3:
  32         return base64.decodebytes(six.ensure_binary(s))
  33     return base64.decodestring(s)
  34
  35
  36 OUTPUT_TEMPLATE = """\
  37 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
  38 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
  39 /* This Source Code Form is subject to the terms of the Mozilla Public
  40  * License, v. 2.0. If a copy of the MPL was not distributed with this
  41  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
  42
  43 /* This file was automatically generated by $prog. */
  44
  45 #ifndef $include_guard
  46 #define $include_guard
  47
  48 #include "CTLog.h"
  49
  50 #include <stddef.h>
  51
  52 struct CTLogInfo
  53 {
  54   // See bug 1338873 about making these fields const.
  55   const char* name;
  56   // Index within kCTLogOperatorList.
  57   mozilla::ct::CTLogStatus status;
  58   // 0 for qualified logs, disqualification time for disqualified logs
  59   // (in milliseconds, measured since the epoch, ignoring leap seconds).
  60   uint64_t disqualificationTime;
  61   size_t operatorIndex;
  62   const char* key;
  63   size_t keyLength;
  64 };
  65
  66 struct CTLogOperatorInfo
  67 {
  68   // See bug 1338873 about making these fields const.
  69   const char* name;
  70   mozilla::ct::CTLogOperatorId id;
  71 };
  72
  73 const CTLogInfo kCTLogList[] = {
  74 $logs
  75 };
  76
  77 const CTLogOperatorInfo kCTLogOperatorList[] = {
  78 $operators
  79 };
  80
  81 #endif // $include_guard
  82 """
  83
  84
  85 def get_disqualification_time(time_str):
  86     """
  87     Convert a time string such as "2017-01-01T00:00:00Z" to an integer
  88     representing milliseconds since the epoch.
  89     Timezones in the string are not supported and will result in an exception.
  90     """
  91     t = datetime.datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ")
  92     epoch = datetime.datetime.utcfromtimestamp(0)
  93     seconds_since_epoch = (t - epoch).total_seconds()
  94     return int(seconds_since_epoch * 1000)
  95
  96
  97 def get_hex_lines(blob, width):
  98     """Convert a binary string to a multiline text of C escape sequences."""
  99     text = "".join(["\\x{:02x}".format(c) for c in blob])
 100     # When escaped, a single byte takes 4 chars (e.g. "\x00").
 101     # Make sure we don't break an escaped byte between the lines.
 102     return textwrap.wrap(text, width - width % 4)
 103
 104
 105 def get_operator_index(json_data, target_name):
 106     """Return operator's entry from the JSON along with its array index."""
 107     matches = [
 108         (operator, index)
 109         for (index, operator) in enumerate(json_data["operators"])
 110         if operator["name"] == target_name
 111     ]
 112     assert len(matches) != 0, "No operators with id {0} defined.".format(target_name)
 113     assert len(matches) == 1, "Found multiple operators with id {0}.".format(
 114         target_name
 115     )
 116     return matches[0][1]
 117
 118
 119 def get_log_info_structs(json_data):
 120     """Return array of CTLogInfo initializers for the known logs."""
 121     tmpl = Template(
 122         textwrap.dedent(
 123             """\
 124           { $description,
 125             $status,
 126             $disqualification_time, // $disqualification_time_comment
 127             $operator_index, // $operator_comment
 128         $indented_log_key,
 129             $log_key_len }"""
 130         )
 131     )
 132     initializers = []
 133     for operator in json_data["operators"]:
 134         operator_name = operator["name"]
 135         for log in operator["logs"]:
 136             log_key = decodebytes(log["key"])
 137             operator_index = get_operator_index(json_data, operator_name)
 138             if "disqualification_time" in log:
 139                 status = "mozilla::ct::CTLogStatus::Disqualified"
 140                 disqualification_time = get_disqualification_time(
 141                     log["disqualification_time"]
 142                 )
 143                 disqualification_time_comment = 'Date.parse("{0}")'.format(
 144                     log["disqualification_time"]
 145                 )
 146             else:
 147                 status = "mozilla::ct::CTLogStatus::Included"
 148                 disqualification_time = 0
 149                 disqualification_time_comment = "no disqualification time"
 150             is_test_log = "test_only" in operator and operator["test_only"]
 151             prefix = ""
 152             suffix = ","
 153             if is_test_log:
 154                 prefix = "#ifdef DEBUG\n"
 155                 suffix = ",\n#endif // DEBUG"
 156             toappend = tmpl.substitute(
 157                 # Use json.dumps for C-escaping strings.
 158                 # Not perfect but close enough.
 159                 description=json.dumps(log["description"]),
 160                 operator_index=operator_index,
 161                 operator_comment="operated by {0}".
 162                 # The comment must not contain "/".
 163                 format(operator_name).replace("/", "|"),
 164                 status=status,
 165                 disqualification_time=disqualification_time,
 166                 disqualification_time_comment=disqualification_time_comment,
 167                 # Maximum line width is 80.
 168                 indented_log_key="\n".join(
 169                     ['    "{0}"'.format(l) for l in get_hex_lines(log_key, 74)]
 170                 ),
 171                 log_key_len=len(log_key),
 172             )
 173             initializers.append(prefix + toappend + suffix)
 174     return initializers
 175
 176
 177 def get_log_operator_structs(json_data):
 178     """Return array of CTLogOperatorInfo initializers."""
 179     tmpl = Template("  { $name, $id }")
 180     initializers = []
 181     currentId = 0
 182     for operator in json_data["operators"]:
 183         prefix = ""
 184         suffix = ","
 185         is_test_log = "test_only" in operator and operator["test_only"]
 186         if is_test_log:
 187             prefix = "#ifdef DEBUG\n"
 188             suffix = ",\n#endif // DEBUG"
 189         toappend = tmpl.substitute(name=json.dumps(operator["name"]), id=currentId)
 190         currentId += 1
 191         initializers.append(prefix + toappend + suffix)
 192     return initializers
 193
 194
 195 def generate_cpp_header_file(json_data, out_file):
 196     """Generate the C++ header file for the known logs."""
 197     filename = os.path.basename(out_file.name)
 198     include_guard = filename.replace(".", "_").replace("/", "_")
 199     log_info_initializers = get_log_info_structs(json_data)
 200     operator_info_initializers = get_log_operator_structs(json_data)
 201     out_file.write(
 202         Template(OUTPUT_TEMPLATE).substitute(
 203             prog=os.path.basename(sys.argv[0]),
 204             include_guard=include_guard,
 205             logs="\n".join(log_info_initializers),
 206             operators="\n".join(operator_info_initializers),
 207         )
 208     )
 209
 210
 211 def patch_in_test_logs(json_data):
 212     """Insert Mozilla-specific test log data."""
 213     max_id = len(json_data["operators"])
 214     mozilla_test_operator_1 = {
 215         "name": "Mozilla Test Org 1",
 216         "id": max_id + 1,
 217         "test_only": True,
 218         "logs": [
 219             {
 220                 "description": "Mozilla Test RSA Log 1",
 221                 # `openssl x509 -noout -pubkey -in <path/to/default-ee.pem>`
 222                 "key": """
 223             MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuohRqESOFtZB/W62iAY2
 224             ED08E9nq5DVKtOz1aFdsJHvBxyWo4NgfvbGcBptuGobya+KvWnVramRxCHqlWqdF
 225             h/cc1SScAn7NQ/weadA4ICmTqyDDSeTbuUzCa2wO7RWCD/F+rWkasdMCOosqQe6n
 226             cOAPDY39ZgsrsCSSpH25iGF5kLFXkD3SO8XguEgfqDfTiEPvJxbYVbdmWqp+ApAv
 227             OnsQgAYkzBxsl62WYVu34pYSwHUxowyR3bTK9/ytHSXTCe+5Fw6naOGzey8ib2nj
 228             tIqVYR3uJtYlnauRCE42yxwkBCy/Fosv5fGPmRcxuLP+SSP6clHEMdUDrNoYCjXt
 229             jQIDAQAB
 230         """,
 231                 "operated_by": [max_id + 1],
 232             },
 233             {
 234                 "description": "Mozilla Test EC Log",
 235                 # `openssl x509 -noout -pubkey -in <path/to/root_secp256r1_256.pem`
 236                 "key": """
 237             MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAET7+7u2Hg+PmxpgpZrIcE4uwFC0I+
 238             PPcukj8sT3lLRVwqadIzRWw2xBGdBwbgDu3I0ZOQ15kbey0HowTqoEqmwA==
 239         """,
 240                 "operated_by": [max_id + 1],
 241             },
 242         ],
 243     }
 244     mozilla_test_operator_2 = {
 245         "name": "Mozilla Test Org 2",
 246         "id": max_id + 2,
 247         "test_only": True,
 248         "logs": [
 249             {
 250                 "description": "Mozilla Test RSA Log 2",
 251                 # `openssl x509 -noout -pubkey -in <path/to/other-test-ca.pem>`
 252                 "key": """
 253             MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXXGUmYJn3cIKmeR8bh2
 254             w39c5TiwbErNIrHL1G+mWtoq3UHIwkmKxKOzwfYUh/QbaYlBvYClHDwSAkTFhKTE
 255             SDMF5ROMAQbPCL6ahidguuai6PNvI8XZgxO53683g0XazlHU1tzSpss8xwbrzTBw
 256             7JjM5AqlkdcpWn9xxb5maR0rLf7ISURZC8Wj6kn9k7HXU0BfF3N2mZWGZiVHl+1C
 257             aQiICBFCIGmYikP+5Izmh4HdIramnNKDdRMfkysSjOKG+n0lHAYq0n7wFvGHzdVO
 258             gys1uJMPdLqQqovHYWckKrH9bWIUDRjEwLjGj8N0hFcyStfehuZVLx0eGR1xIWjT
 259             uwIDAQAB
 260         """,
 261                 "operated_by": [max_id + 2],
 262             }
 263         ],
 264     }
 265     json_data["operators"].append(mozilla_test_operator_1)
 266     json_data["operators"].append(mozilla_test_operator_2)
 267
 268
 269 def run(args):
 270     """
 271     Load the input JSON file and generate the C++ header according to the
 272     command line arguments.
 273     """
 274     if args.file:
 275         print("Reading file: ", args.file)
 276         with open(args.file, "rb") as json_file:
 277             json_text = json_file.read()
 278     elif args.url:
 279         print("Fetching URL: ", args.url)
 280         json_request = urllib3.urlopen(args.url)
 281         try:
 282             json_text = json_request.read()
 283         finally:
 284             json_request.close()
 285
 286     json_data = json.loads(json_text)
 287
 288     print("Writing output: ", args.out)
 289
 290     patch_in_test_logs(json_data)
 291
 292     with open(args.out, "w") as out_file:
 293         generate_cpp_header_file(json_data, out_file)
 294
 295     print("Done.")
 296
 297
 298 def parse_arguments_and_run():
 299     """Parse the command line arguments and run the program."""
 300     arg_parser = argparse.ArgumentParser(
 301         description="Parses a JSON file listing the known "
 302         "Certificate Transparency logs and generates "
 303         "a C++ header file to be included in Firefox.",
 304         epilog="Example: python %s --url" % os.path.basename(sys.argv[0]),
 305     )
 306
 307     source_group = arg_parser.add_mutually_exclusive_group(required=True)
 308     source_group.add_argument(
 309         "--file",
 310         nargs="?",
 311         const="log_list.json",
 312         help="Read the known CT logs JSON data from the "
 313         "specified local file (%(const)s by default).",
 314     )
 315     source_group.add_argument(
 316         "--url", help="Download the known CT logs JSON file " "from the specified URL."
 317     )
 318
 319     arg_parser.add_argument(
 320         "--out",
 321         default="../../certverifier/CTKnownLogs.h",
 322         help="Path and filename of the header file "
 323         "to be generated. Defaults to %(default)s",
 324     )
 325
 326     run(arg_parser.parse_args())
 327
 328
 329 if __name__ == "__main__":
 330     parse_arguments_and_run()