build/midl.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import functools
   6 import os
   7 import shutil
   8 import subprocess
   9 import sys
  10
  11 import buildconfig
  12
  13
  14 def relativize(path, base=None):
  15     # For absolute path in Unix builds, we need relative paths because
  16     # Windows programs run via Wine don't like these Unix absolute paths
  17     # (they look like command line arguments).
  18     if path.startswith("/"):
  19         return os.path.relpath(path, base)
  20     # For Windows absolute paths, we can just use the unmodified path.
  21     # And if the path starts with '-', it's a command line argument.
  22     if os.path.isabs(path) or path.startswith("-"):
  23         return path
  24     # Remaining case is relative paths, which may be relative to a different
  25     # directory (os.getcwd()) than the needed `base`, so we "rebase" it.
  26     return os.path.relpath(path, base)
  27
  28
  29 @functools.lru_cache(maxsize=None)
  30 def files_in(path):
  31     return {p.lower(): os.path.join(path, p) for p in os.listdir(path)}
  32
  33
  34 def search_path(paths, path):
  35     for p in paths:
  36         f = os.path.join(p, path)
  37         if os.path.isfile(f):
  38             return f
  39         # try an case-insensitive match
  40         maybe_match = files_in(p).get(path.lower())
  41         if maybe_match:
  42             return maybe_match
  43     raise RuntimeError(f"Cannot find {path}")
  44
  45
  46 # Filter-out -std= flag from the preprocessor command, as we're not preprocessing
  47 # C or C++, and the command would fail with the flag.
  48 def filter_preprocessor(cmd):
  49     prev = None
  50     for arg in cmd:
  51         if arg == "-Xclang":
  52             prev = arg
  53             continue
  54         if not arg.startswith("-std="):
  55             if prev:
  56                 yield prev
  57             yield arg
  58         prev = None
  59
  60
  61 # Preprocess all the direct and indirect inputs of midl, and put all the
  62 # preprocessed inputs in the given `base` directory. Returns a tuple containing
  63 # the path of the main preprocessed input, and the modified flags to use instead
  64 # of the flags given as argument.
  65 def preprocess(base, input, flags):
  66     import argparse
  67     import re
  68     from collections import deque
  69
  70     IMPORT_RE = re.compile('import\s*"([^"]+)";')
  71
  72     parser = argparse.ArgumentParser()
  73     parser.add_argument("-I", action="append")
  74     parser.add_argument("-D", action="append")
  75     parser.add_argument("-acf")
  76     args, remainder = parser.parse_known_args(flags)
  77     preprocessor = (
  78         list(filter_preprocessor(buildconfig.substs["CXXCPP"]))
  79         # Ideally we'd use the real midl version, but querying it adds a
  80         # significant overhead to configure. In practice, the version number
  81         # doesn't make a difference at the moment.
  82         + ["-D__midl=801"]
  83         + [f"-D{d}" for d in args.D or ()]
  84         + [f"-I{i}" for i in args.I or ()]
  85     )
  86     includes = ["."] + buildconfig.substs["INCLUDE"].split(";") + (args.I or [])
  87     seen = set()
  88     queue = deque([input])
  89     if args.acf:
  90         queue.append(args.acf)
  91     output = os.path.join(base, os.path.basename(input))
  92     while True:
  93         try:
  94             input = queue.popleft()
  95         except IndexError:
  96             break
  97         if os.path.basename(input) in seen:
  98             continue
  99         seen.add(os.path.basename(input))
 100         input = search_path(includes, input)
 101         # If there is a .acf file corresponding to the .idl we're processing,
 102         # we also want to preprocess that file because midl might look for it too.
 103         if input.lower().endswith(".idl"):
 104             try:
 105                 acf = search_path(
 106                     [os.path.dirname(input)], os.path.basename(input)[:-4] + ".acf"
 107                 )
 108                 if acf:
 109                     queue.append(acf)
 110             except RuntimeError:
 111                 pass
 112         command = preprocessor + [input]
 113         preprocessed = os.path.join(base, os.path.basename(input))
 114         subprocess.run(command, stdout=open(preprocessed, "wb"), check=True)
 115         # Read the resulting file, and search for imports, that we'll want to
 116         # preprocess as well.
 117         with open(preprocessed, "r") as fh:
 118             for line in fh:
 119                 if not line.startswith("import"):
 120                     continue
 121                 m = IMPORT_RE.match(line)
 122                 if not m:
 123                     continue
 124                 imp = m.group(1)
 125                 queue.append(imp)
 126     flags = []
 127     # Add -I<base> first in the flags, so that midl resolves imports to the
 128     # preprocessed files we created.
 129     for i in [base] + (args.I or []):
 130         flags.extend(["-I", i])
 131     # Add the preprocessed acf file if one was given on the command line.
 132     if args.acf:
 133         flags.extend(["-acf", os.path.join(base, os.path.basename(args.acf))])
 134     flags.extend(remainder)
 135     return output, flags
 136
 137
 138 def midl(out, input, *flags):
 139     out.avoid_writing_to_file()
 140     midl_flags = buildconfig.substs["MIDL_FLAGS"]
 141     base = os.path.dirname(out.name) or "."
 142     tmpdir = None
 143     try:
 144         # If the build system is asking to not use the preprocessor to midl,
 145         # we need to do the preprocessing ourselves.
 146         if "-no_cpp" in midl_flags:
 147             # Normally, we'd use tempfile.TemporaryDirectory, but in this specific
 148             # case, we actually want a deterministic directory name, because it's
 149             # recorded in the code midl generates.
 150             tmpdir = os.path.join(base, os.path.basename(input) + ".tmp")
 151             os.makedirs(tmpdir, exist_ok=True)
 152             try:
 153                 input, flags = preprocess(tmpdir, input, flags)
 154             except subprocess.CalledProcessError as e:
 155                 return e.returncode
 156         midl = buildconfig.substs["MIDL"]
 157         wine = buildconfig.substs.get("WINE")
 158         if midl.lower().endswith(".exe") and wine:
 159             command = [wine, midl]
 160         else:
 161             command = [midl]
 162         command.extend(midl_flags)
 163         command.extend([relativize(f, base) for f in flags])
 164         command.append("-Oicf")
 165         command.append(relativize(input, base))
 166         print("Executing:", " ".join(command))
 167         result = subprocess.run(command, cwd=base)
 168         return result.returncode
 169     finally:
 170         if tmpdir:
 171             shutil.rmtree(tmpdir)
 172
 173
 174 # midl outputs dlldata to a single dlldata.c file by default. This prevents running
 175 # midl in parallel in the same directory for idl files that would generate dlldata.c
 176 # because of race conditions updating the file. Instead, we ask midl to create
 177 # separate files, and we merge them manually.
 178 def merge_dlldata(out, *inputs):
 179     inputs = [open(i) for i in inputs]
 180     read_a_line = [True] * len(inputs)
 181     while True:
 182         lines = [
 183             f.readline() if read_a_line[n] else lines[n] for n, f in enumerate(inputs)
 184         ]
 185         unique_lines = set(lines)
 186         if len(unique_lines) == 1:
 187             # All the lines are identical
 188             if not lines[0]:
 189                 break
 190             out.write(lines[0])
 191             read_a_line = [True] * len(inputs)
 192         elif (
 193             len(unique_lines) == 2
 194             and len([l for l in unique_lines if "#define" in l]) == 1
 195         ):
 196             # Most lines are identical. When they aren't, it's typically because some
 197             # files have an extra #define that others don't. When that happens, we
 198             # print out the #define, and get a new input line from the files that had
 199             # a #define on the next iteration. We expect that next line to match what
 200             # the other files had on this iteration.
 201             # Note: we explicitly don't support the case where there are different
 202             # defines across different files, except when there's a different one
 203             # for each file, in which case it's handled further below.
 204             a = unique_lines.pop()
 205             if "#define" in a:
 206                 out.write(a)
 207             else:
 208                 out.write(unique_lines.pop())
 209             read_a_line = ["#define" in l for l in lines]
 210         elif len(unique_lines) != len(lines):
 211             # If for some reason, we don't get lines that are entirely different
 212             # from each other, we have some unexpected input.
 213             print(
 214                 "Error while merging dlldata. Last lines read: {}".format(lines),
 215                 file=sys.stderr,
 216             )
 217             return 1
 218         else:
 219             for line in lines:
 220                 out.write(line)
 221             read_a_line = [True] * len(inputs)
 222
 223     return 0