build/midl.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import buildconfig
   6 import shutil
   7 import subprocess
   8 import os
   9 import sys
  10
  11
  12 def relativize(path, base=None):
  13     # For absolute path in Unix builds, we need relative paths because
  14     # Windows programs run via Wine don't like these Unix absolute paths
  15     # (they look like command line arguments).
  16     if path.startswith("/"):
  17         return os.path.relpath(path, base)
  18     # For Windows absolute paths, we can just use the unmodified path.
  19     # And if the path starts with '-', it's a command line argument.
  20     if os.path.isabs(path) or path.startswith("-"):
  21         return path
  22     # Remaining case is relative paths, which may be relative to a different
  23     # directory (os.getcwd()) than the needed `base`, so we "rebase" it.
  24     return os.path.relpath(path, base)
  25
  26
  27 def search_path(paths, path):
  28     for p in paths:
  29         f = os.path.join(p, path)
  30         if os.path.isfile(f):
  31             return f
  32     raise RuntimeError(f"Cannot find {path}")
  33
  34
  35 # Preprocess all the direct and indirect inputs of midl, and put all the
  36 # preprocessed inputs in the given `base` directory. Returns a tuple containing
  37 # the path of the main preprocessed input, and the modified flags to use instead
  38 # of the flags given as argument.
  39 def preprocess(base, input, flags):
  40     import argparse
  41     import re
  42     from collections import deque
  43
  44     IMPORT_RE = re.compile('import\s*"([^"]+)";')
  45
  46     parser = argparse.ArgumentParser()
  47     parser.add_argument("-I", action="append")
  48     parser.add_argument("-D", action="append")
  49     parser.add_argument("-acf")
  50     args, remainder = parser.parse_known_args(flags)
  51     preprocessor = (
  52         [buildconfig.substs["_CXX"]]
  53         # Ideally we'd use the real midl version, but querying it adds a
  54         # significant overhead to configure. In practice, the version number
  55         # doesn't make a difference at the moment.
  56         + ["-E", "-D__midl=801"]
  57         + [f"-D{d}" for d in args.D or ()]
  58         + [f"-I{i}" for i in args.I or ()]
  59     )
  60     includes = ["."] + buildconfig.substs["INCLUDE"].split(";") + (args.I or [])
  61     seen = set()
  62     queue = deque([input])
  63     if args.acf:
  64         queue.append(args.acf)
  65     output = os.path.join(base, os.path.basename(input))
  66     while True:
  67         try:
  68             input = queue.popleft()
  69         except IndexError:
  70             break
  71         if os.path.basename(input) in seen:
  72             continue
  73         seen.add(os.path.basename(input))
  74         input = search_path(includes, input)
  75         # If there is a .acf file corresponding to the .idl we're processing,
  76         # we also want to preprocess that file because midl might look for it too.
  77         if input.endswith(".idl") and os.path.exists(input[:-4] + ".acf"):
  78             queue.append(input[:-4] + ".acf")
  79         command = preprocessor + [input]
  80         preprocessed = os.path.join(base, os.path.basename(input))
  81         subprocess.run(command, stdout=open(preprocessed, "wb"), check=True)
  82         # Read the resulting file, and search for imports, that we'll want to
  83         # preprocess as well.
  84         with open(preprocessed, "r") as fh:
  85             for line in fh:
  86                 if not line.startswith("import"):
  87                     continue
  88                 m = IMPORT_RE.match(line)
  89                 if not m:
  90                     continue
  91                 imp = m.group(1)
  92                 queue.append(imp)
  93     flags = []
  94     # Add -I<base> first in the flags, so that midl resolves imports to the
  95     # preprocessed files we created.
  96     for i in [base] + (args.I or []):
  97         flags.extend(["-I", i])
  98     # Add the preprocessed acf file if one was given on the command line.
  99     if args.acf:
 100         flags.extend(["-acf", os.path.join(base, os.path.basename(args.acf))])
 101     flags.extend(remainder)
 102     return output, flags
 103
 104
 105 def midl(out, input, *flags):
 106     out.avoid_writing_to_file()
 107     midl_flags = buildconfig.substs["MIDL_FLAGS"]
 108     base = os.path.dirname(out.name) or "."
 109     tmpdir = None
 110     try:
 111         # If the build system is asking to not use the preprocessor to midl,
 112         # we need to do the preprocessing ourselves.
 113         if "-no_cpp" in midl_flags:
 114             # Normally, we'd use tempfile.TemporaryDirectory, but in this specific
 115             # case, we actually want a deterministic directory name, because it's
 116             # recorded in the code midl generates.
 117             tmpdir = os.path.join(base, os.path.basename(input) + ".tmp")
 118             os.makedirs(tmpdir, exist_ok=True)
 119             try:
 120                 input, flags = preprocess(tmpdir, input, flags)
 121             except subprocess.CalledProcessError as e:
 122                 return e.returncode
 123         midl = buildconfig.substs["MIDL"]
 124         wine = buildconfig.substs.get("WINE")
 125         if midl.lower().endswith(".exe") and wine:
 126             command = [wine, midl]
 127         else:
 128             command = [midl]
 129         command.extend(midl_flags)
 130         command.extend([relativize(f, base) for f in flags])
 131         command.append("-Oicf")
 132         command.append(relativize(input, base))
 133         print("Executing:", " ".join(command))
 134         result = subprocess.run(command, cwd=base)
 135         return result.returncode
 136     finally:
 137         if tmpdir:
 138             shutil.rmtree(tmpdir)
 139
 140
 141 # midl outputs dlldata to a single dlldata.c file by default. This prevents running
 142 # midl in parallel in the same directory for idl files that would generate dlldata.c
 143 # because of race conditions updating the file. Instead, we ask midl to create
 144 # separate files, and we merge them manually.
 145 def merge_dlldata(out, *inputs):
 146     inputs = [open(i) for i in inputs]
 147     read_a_line = [True] * len(inputs)
 148     while True:
 149         lines = [
 150             f.readline() if read_a_line[n] else lines[n] for n, f in enumerate(inputs)
 151         ]
 152         unique_lines = set(lines)
 153         if len(unique_lines) == 1:
 154             # All the lines are identical
 155             if not lines[0]:
 156                 break
 157             out.write(lines[0])
 158             read_a_line = [True] * len(inputs)
 159         elif (
 160             len(unique_lines) == 2
 161             and len([l for l in unique_lines if "#define" in l]) == 1
 162         ):
 163             # Most lines are identical. When they aren't, it's typically because some
 164             # files have an extra #define that others don't. When that happens, we
 165             # print out the #define, and get a new input line from the files that had
 166             # a #define on the next iteration. We expect that next line to match what
 167             # the other files had on this iteration.
 168             # Note: we explicitly don't support the case where there are different
 169             # defines across different files, except when there's a different one
 170             # for each file, in which case it's handled further below.
 171             a = unique_lines.pop()
 172             if "#define" in a:
 173                 out.write(a)
 174             else:
 175                 out.write(unique_lines.pop())
 176             read_a_line = ["#define" in l for l in lines]
 177         elif len(unique_lines) != len(lines):
 178             # If for some reason, we don't get lines that are entirely different
 179             # from each other, we have some unexpected input.
 180             print(
 181                 "Error while merging dlldata. Last lines read: {}".format(lines),
 182                 file=sys.stderr,
 183             )
 184             return 1
 185         else:
 186             for line in lines:
 187                 out.write(line)
 188             read_a_line = [True] * len(inputs)
 189
 190     return 0