tools/moztreedocs/__init__.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 from pathlib import PurePath
   7
   8 import sphinx
   9 import sphinx.ext.apidoc
  10 import yaml
  11 from mozbuild.base import MozbuildObject
  12 from mozbuild.frontend.reader import BuildReader
  13 from mozbuild.util import memoize
  14 from mozpack.copier import FileCopier
  15 from mozpack.files import FileFinder
  16 from mozpack.manifests import InstallManifest
  17
  18 here = os.path.abspath(os.path.dirname(__file__))
  19 build = MozbuildObject.from_environment(cwd=here)
  20
  21 MAIN_DOC_PATH = os.path.normpath(os.path.join(build.topsrcdir, "docs"))
  22
  23 logger = sphinx.util.logging.getLogger(__name__)
  24
  25
  26 @memoize
  27 def read_build_config(docdir):
  28     """Read the active build config and return the relevant doc paths.
  29
  30     The return value is cached so re-generating with the same docdir won't
  31     invoke the build system a second time."""
  32     trees = {}
  33     python_package_dirs = set()
  34
  35     is_main = docdir == MAIN_DOC_PATH
  36     relevant_mozbuild_path = None if is_main else docdir
  37
  38     # Reading the Sphinx variables doesn't require a full build context.
  39     # Only define the parts we need.
  40     class fakeconfig(object):
  41         topsrcdir = build.topsrcdir
  42
  43     variables = ("SPHINX_TREES", "SPHINX_PYTHON_PACKAGE_DIRS")
  44     reader = BuildReader(fakeconfig())
  45     result = reader.find_variables_from_ast(variables, path=relevant_mozbuild_path)
  46     for path, name, key, value in result:
  47         reldir = os.path.dirname(path)
  48
  49         if name == "SPHINX_TREES":
  50             # If we're building a subtree, only process that specific subtree.
  51             # topsrcdir always uses POSIX-style path, normalize it for proper comparison.
  52             absdir = os.path.normpath(os.path.join(build.topsrcdir, reldir, value))
  53             if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
  54                 # allow subpaths of absdir (i.e. docdir = <absdir>/sub/path/)
  55                 if docdir.startswith(absdir):
  56                     key = os.path.join(key, docdir.split(f"{key}/")[-1])
  57                 else:
  58                     continue
  59
  60             assert key
  61             if key.startswith("/"):
  62                 key = key[1:]
  63             else:
  64                 key = os.path.normpath(os.path.join(reldir, key))
  65
  66             if key in trees:
  67                 raise Exception(
  68                     "%s has already been registered as a destination." % key
  69                 )
  70             trees[key] = os.path.join(reldir, value)
  71
  72         if name == "SPHINX_PYTHON_PACKAGE_DIRS":
  73             python_package_dirs.add(os.path.join(reldir, value))
  74
  75     return trees, python_package_dirs
  76
  77
  78 class _SphinxManager(object):
  79     """Manages the generation of Sphinx documentation for the tree."""
  80
  81     NO_AUTODOC = False
  82
  83     def __init__(self, topsrcdir, main_path):
  84         self.topsrcdir = topsrcdir
  85         self.conf_py_path = os.path.join(main_path, "conf.py")
  86         self.index_path = os.path.join(main_path, "index.rst")
  87
  88         # Instance variables that get set in self.generate_docs()
  89         self.staging_dir = None
  90         self.trees = None
  91         self.python_package_dirs = None
  92
  93     def generate_docs(self, app):
  94         """Generate/stage documentation."""
  95         if self.NO_AUTODOC:
  96             logger.info("Python/JS API documentation generation will be skipped")
  97             app.config["extensions"].remove("sphinx.ext.autodoc")
  98             app.config["extensions"].remove("sphinx_js")
  99         self.staging_dir = os.path.join(app.outdir, "_staging")
 100
 101         logger.info("Reading Sphinx metadata from build configuration")
 102         self.trees, self.python_package_dirs = read_build_config(app.srcdir)
 103
 104         logger.info("Staging static documentation")
 105         self._synchronize_docs(app)
 106
 107         if not self.NO_AUTODOC:
 108             self._generate_python_api_docs()
 109
 110     def _generate_python_api_docs(self):
 111         """Generate Python API doc files."""
 112         out_dir = os.path.join(self.staging_dir, "python")
 113         base_args = ["--no-toc", "-o", out_dir]
 114
 115         for p in sorted(self.python_package_dirs):
 116             full = os.path.join(self.topsrcdir, p)
 117
 118             finder = FileFinder(full)
 119             dirs = {os.path.dirname(f[0]) for f in finder.find("**")}
 120
 121             test_dirs = {"test", "tests"}
 122             excludes = {d for d in dirs if set(PurePath(d).parts) & test_dirs}
 123
 124             args = list(base_args)
 125             args.append(full)
 126             args.extend(excludes)
 127
 128             sphinx.ext.apidoc.main(argv=args)
 129
 130     def _synchronize_docs(self, app):
 131         m = InstallManifest()
 132
 133         with open(os.path.join(MAIN_DOC_PATH, "config.yml"), "r") as fh:
 134             tree_config = yaml.safe_load(fh)["categories"]
 135
 136         m.add_link(self.conf_py_path, "conf.py")
 137
 138         for dest, source in sorted(self.trees.items()):
 139             source_dir = os.path.join(self.topsrcdir, source)
 140             for root, _, files in os.walk(source_dir):
 141                 for f in files:
 142                     source_path = os.path.normpath(os.path.join(root, f))
 143                     rel_source = source_path[len(source_dir) + 1 :]
 144                     target = os.path.normpath(os.path.join(dest, rel_source))
 145                     m.add_link(source_path, target)
 146
 147         copier = FileCopier()
 148         m.populate_registry(copier)
 149
 150         # In the case of livereload, we don't want to delete unmodified (unaccounted) files.
 151         copier.copy(
 152             self.staging_dir, remove_empty_directories=False, remove_unaccounted=False
 153         )
 154
 155         with open(self.index_path, "r") as fh:
 156             data = fh.read()
 157
 158         def is_toplevel(key):
 159             """Whether the tree is nested under the toplevel index, or is
 160             nested under another tree's index.
 161             """
 162             for k in self.trees:
 163                 if k == key:
 164                     continue
 165                 if key.startswith(k):
 166                     return False
 167             return True
 168
 169         def format_paths(paths):
 170             source_doc = ["%s/index" % p for p in paths]
 171             return "\n   ".join(source_doc)
 172
 173         toplevel_trees = {k: v for k, v in self.trees.items() if is_toplevel(k)}
 174
 175         CATEGORIES = {}
 176         # generate the datastructure to deal with the tree
 177         for t in tree_config:
 178             CATEGORIES[t] = format_paths(tree_config[t])
 179
 180         # During livereload, we don't correctly rebuild the full document
 181         # tree (Bug 1557020). The page is no longer referenced within the index
 182         # tree, thus we shall check categorisation only if complete tree is being rebuilt.
 183         if app.srcdir == self.topsrcdir:
 184             indexes = set(
 185                 [
 186                     os.path.normpath(os.path.join(p, "index"))
 187                     for p in toplevel_trees.keys()
 188                 ]
 189             )
 190             # Format categories like indexes
 191             cats = "\n".join(CATEGORIES.values()).split("\n")
 192             # Remove heading spaces
 193             cats = [os.path.normpath(x.strip()) for x in cats]
 194             indexes = tuple(set(indexes) - set(cats))
 195             if indexes:
 196                 # In case a new doc isn't categorized
 197                 print(indexes)
 198                 raise Exception(
 199                     "Uncategorized documentation. Please add it in docs/config.yml"
 200                 )
 201
 202         data = data.format(**CATEGORIES)
 203
 204         with open(os.path.join(self.staging_dir, "index.rst"), "w") as fh:
 205             fh.write(data)
 206
 207
 208 manager = _SphinxManager(build.topsrcdir, MAIN_DOC_PATH)