pygnulib/GLTestDir.py

   1 # Copyright (C) 2002-2024 Free Software Foundation, Inc.
   2 #
   3 # This program is free software: you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation, either version 3 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  15
  16 from __future__ import annotations
  17
  18 #===============================================================================
  19 # Define global imports
  20 #===============================================================================
  21 import os
  22 import re
  23 import sys
  24 import shlex
  25 import subprocess as sp
  26 from pathlib import Path
  27 from .constants import (
  28     DIRS,
  29     TESTS,
  30     UTILS,
  31     combine_lines,
  32     execute,
  33     ensure_writable,
  34     force_output,
  35     hardlink,
  36     joinpath,
  37     link_relative,
  38     lines_to_multiline,
  39     movefile,
  40     copyfile,
  41     substart,
  42     bold_escapes,
  43     relinverse,
  44     rmtree,
  45 )
  46 from .functions import rewrite_file_name
  47 from .enums import CopyAction
  48 from .GLError import GLError
  49 from .GLConfig import GLConfig
  50 from .GLModuleSystem import GLModuleTable, GLModuleSystem
  51 from .GLFileSystem import GLFileSystem
  52 from .GLMakefileTable import GLMakefileTable
  53 from .GLEmiter import GLEmiter
  54 from .GLFileTable import GLFileTable
  55
  56
  57 def _patch_test_driver() -> None:
  58     '''Patch the test-driver script in testdirs.'''
  59     test_driver = joinpath('build-aux', 'test-driver')
  60     print('patching file %s' % test_driver)
  61     diffs = [ joinpath(DIRS['root'], name)
  62               for name in [joinpath('build-aux', 'test-driver.diff'),
  63                            joinpath('build-aux', 'test-driver-1.16.3.diff')] ]
  64     patched = False
  65     for diff in diffs:
  66         command = f'patch {shlex.quote(test_driver)} < {shlex.quote(diff)}'
  67         try:
  68             result = sp.call(command, shell=True, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
  69         except OSError as exc:
  70             if os.path.isfile(f'{test_driver}.orig'):
  71                 os.remove(f'{test_driver}.orig')
  72             if os.path.isfile(f'{test_driver}.rej'):
  73                 os.remove(f'{test_driver}.rej')
  74             raise GLError(20, None) from exc
  75         if result == 0:
  76             patched = True
  77             break
  78         if os.path.isfile(f'{test_driver}.orig'):
  79             os.remove(f'{test_driver}.orig')
  80         if os.path.isfile(f'{test_driver}.rej'):
  81             os.remove(f'{test_driver}.rej')
  82     if not patched:
  83         raise GLError(20, None)
  84
  85
  86 #===============================================================================
  87 # Define GLTestDir class
  88 #===============================================================================
  89 class GLTestDir:
  90     '''GLTestDir class is used to create a scratch package with the given
  91     list of the modules.'''
  92
  93     config: GLConfig
  94     testdir: str
  95     emitter: GLEmiter
  96     filesystem: GLFileSystem
  97     modulesystem: GLModuleSystem
  98     makefiletable: GLMakefileTable
  99
 100     def __init__(self, config: GLConfig, testdir: str) -> None:
 101         '''Create new GLTestDir instance.'''
 102         if type(config) is not GLConfig:
 103             raise TypeError('config must be a GLConfig, not %s'
 104                             % type(config).__name__)
 105         if type(testdir) is not str:
 106             raise TypeError('testdir must be a string, not %s'
 107                             % type(testdir).__name__)
 108         self.config = config
 109         self.testdir = os.path.normpath(testdir)
 110         # Don't overwrite the directory.
 111         if os.path.exists(self.testdir):
 112             raise GLError(22, self.testdir)
 113         # Try to create directory.
 114         try:
 115             os.mkdir(self.testdir)
 116         except Exception as exc:
 117             raise GLError(19, self.testdir) from exc
 118         self.emitter = GLEmiter(self.config)
 119         self.filesystem = GLFileSystem(self.config)
 120         self.modulesystem = GLModuleSystem(self.config)
 121         self.makefiletable = GLMakefileTable(self.config)
 122
 123         # Subdirectory names.
 124         self.config.setSourceBase('gllib')
 125         self.config.setM4Base('glm4')
 126         self.config.setDocBase('gldoc')
 127         self.config.setTestsBase('gltests')
 128         self.config.setMacroPrefix('gl')
 129         self.config.resetPoBase()
 130         self.config.resetPoDomain()
 131         self.config.resetWitnessCMacro()
 132         self.config.resetVCFiles()
 133
 134     def execute(self) -> None:
 135         '''Create a scratch package with the given modules.'''
 136         auxdir = self.config['auxdir']
 137         sourcebase = self.config['sourcebase']
 138         m4base = self.config['m4base']
 139         testsbase = self.config['testsbase']
 140         libname = self.config['libname']
 141         libtool = self.config['libtool']
 142         single_configure = self.config['single_configure']
 143         macro_prefix = self.config['macro_prefix']
 144         verbose = self.config['verbosity']
 145
 146         specified_modules = self.config['modules']
 147         if len(specified_modules) == 0:
 148             # All modules together.
 149             # Except config-h, which breaks all modules which use HAVE_CONFIG_H.
 150             # Except non-recursive-gnulib-prefix-hack, which represents a nonstandard
 151             # way of using Automake.
 152             # Except timevar, which lacks the required file timevar.def.
 153             # Except mountlist, which aborts the configuration on mingw. FIXME.
 154             # Except lib-ignore, which leads to link errors when Sun C++ is used. FIXME.
 155             specified_modules = self.modulesystem.list()
 156             specified_modules = [module
 157                                  for module in specified_modules
 158                                  if module not in ['config-h', 'non-recursive-gnulib-prefix-hack', 'timevar',
 159                                                    'mountlist', 'lib-ignore']]
 160
 161         # Canonicalize the list of specified modules.
 162         modules = set()
 163         for name in specified_modules:
 164             module = self.modulesystem.find(name)
 165             if module is not None:
 166                 modules.add(module)
 167         specified_modules = sorted(modules)
 168
 169         # Test modules which invoke AC_CONFIG_FILES cannot be used with
 170         # --with-tests --single-configure. Avoid them.
 171         inctests = self.config.checkInclTestCategory(TESTS['tests'])
 172         if inctests and self.config.checkSingleConfigure():
 173             self.config.addAvoid('havelib-tests')
 174
 175         # Now it's time to create the GLModuleTable.
 176         moduletable = GLModuleTable(self.config,
 177                                     True,
 178                                     self.config.checkInclTestCategory(TESTS['all-tests']))
 179
 180         # When computing transitive closures, don't consider $module to depend on
 181         # $module-tests. Need this because tests are implicitly GPL and may depend
 182         # on GPL modules - therefore we don't want a warning in this case.
 183         saved_inctests = self.config.checkInclTestCategory(TESTS['tests'])
 184         self.config.disableInclTestCategory(TESTS['tests'])
 185         for requested_module in specified_modules:
 186             requested_licence = requested_module.getLicense()
 187             if requested_licence != 'GPL':
 188                 # Here we use moduletable.transitive_closure([module]), not
 189                 # just module.getDependencies, so that we also detect weird
 190                 # situations like an LGPL module which depends on a GPLed build
 191                 # tool module which depends on a GPL module.
 192                 modules = moduletable.transitive_closure([requested_module])
 193                 for module in modules:
 194                     license = module.getLicense()
 195                     if license not in ['GPLv2+ build tool', 'GPLed build tool',
 196                                        'public domain', 'unlimited', 'unmodifiable license text']:
 197                         incompatible = False
 198                         if requested_licence == 'GPLv3+' or requested_licence == 'GPL':
 199                             if license not in ['LGPLv2+', 'LGPLv3+ or GPLv2+', 'LGPLv3+', 'LGPL', 'GPLv2+', 'GPLv3+', 'GPL']:
 200                                 incompatible = True
 201                         elif requested_licence == 'GPLv2+':
 202                             if license not in ['LGPLv2+', 'LGPLv3+ or GPLv2+', 'GPLv2+']:
 203                                 incompatible = True
 204                         elif requested_licence == 'LGPLv3+' or requested_licence == 'LGPL':
 205                             if license not in ['LGPLv2+', 'LGPLv3+ or GPLv2+', 'LGPLv3+', 'LGPL']:
 206                                 incompatible = True
 207                         elif requested_licence == 'LGPLv3+ or GPLv2+':
 208                             if license not in ['LGPLv2+', 'LGPLv3+ or GPLv2+']:
 209                                 incompatible = True
 210                         elif requested_licence == 'LGPLv2+':
 211                             if license not in ['LGPLv2+']:
 212                                 incompatible = True
 213                         if incompatible:
 214                             warningmsg = 'module %s depends on a module with an incompatible license: %s' % (requested_module, module)
 215                             sys.stderr.write('gnulib-tool: warning: %s\n' % warningmsg)
 216         self.config.setInclTestCategory(TESTS['tests'], saved_inctests)
 217
 218         # Determine final module list.
 219         modules = moduletable.transitive_closure(specified_modules)
 220         final_modules = list(modules)
 221
 222         # Show final module list.
 223         if verbose >= 0:
 224             (bold_on, bold_off) = bold_escapes()
 225             print('Module list with included dependencies (indented):')
 226             specified_modules_set = { module.name
 227                                       for module in specified_modules }
 228             for module in final_modules:
 229                 if module.name in specified_modules_set:
 230                     print('  %s%s%s' % (bold_on, module.name, bold_off))
 231                 else:  # if module.name not in specified_modules_set
 232                     print('    %s' % module.name)
 233
 234         # Generate lists of the modules.
 235         if single_configure:
 236             # Determine main module list and tests-related module list separately.
 237             main_modules, tests_modules = \
 238                 moduletable.transitive_closure_separately(specified_modules, final_modules)
 239             # Print main_modules and tests_modules.
 240             if verbose >= 1:
 241                 print('Main module list:')
 242                 for module in main_modules:
 243                     print('  %s' % module.name)
 244                 print('Tests-related module list:')
 245                 for module in tests_modules:
 246                     print('  %s' % module.name)
 247             # Determine whether a $testsbase/libtests.a is needed.
 248             libtests = False
 249             for module in tests_modules:
 250                 files = module.getFiles()
 251                 for file in files:
 252                     if file.startswith('lib/'):
 253                         libtests = True
 254                         break
 255             self.emitter.config.setLibtests(libtests)
 256
 257         if single_configure:
 258             # Add the dummy module to the main module list if needed.
 259             main_modules = moduletable.add_dummy(main_modules)
 260             if libtests:  # if we need to use libtests.a
 261                 # Add the dummy module to the tests-related module list if needed.
 262                 tests_modules = moduletable.add_dummy(tests_modules)
 263         else:  # if not single_configure
 264             modules = moduletable.add_dummy(modules)
 265
 266         # Show banner notice of every module.
 267         if single_configure:
 268             if verbose >= -1:
 269                 for module in main_modules:
 270                     notice = module.getNotice().strip('\n')
 271                     if notice:
 272                         print('Notice from module %s:' % module.name)
 273                         pattern = re.compile(r'^(.*)$', re.M)
 274                         notice = pattern.sub(r'  \1', notice)
 275                         print(notice)
 276         else:  # if not single_configure
 277             if verbose >= -1:
 278                 for module in modules:
 279                     notice = module.getNotice().strip('\n')
 280                     if notice:
 281                         print('Notice from module %s:' % module.name)
 282                         pattern = re.compile(r'^(.*)$', re.M)
 283                         notice = pattern.sub(r'  \1', notice)
 284                         print(notice)
 285
 286         # Determine final file list.
 287         if single_configure:
 288             main_filelist, tests_filelist = \
 289                 moduletable.filelist_separately(main_modules, tests_modules)
 290         else:  # if not single_configure
 291             main_modules = modules
 292             tests_modules = [ module
 293                               for module in modules
 294                               if module.repeatModuleInTests() ]
 295             main_filelist, tests_filelist = \
 296                 moduletable.filelist_separately(main_modules, tests_modules)
 297
 298         filelist = sorted(set(main_filelist + tests_filelist))
 299
 300         # Print list of files.
 301         if verbose >= 0:
 302             print('File list:')
 303             for file in filelist:
 304                 if file.startswith('tests=lib/'):
 305                     rest = file[10:]
 306                     print('  lib/%s -> tests/%s' % (rest, rest))
 307                 else:
 308                     print('  %s' % file)
 309
 310         # Add files for which the copy in gnulib is newer than the one that
 311         # "automake --add-missing --copy" would provide.
 312         filelist = sorted(set(filelist + ['build-aux/config.guess', 'build-aux/config.sub']))
 313
 314         # new_table is a table with two columns: (rewritten-file-name original-file-name),
 315         # representing the files after this gnulib-tool invocation.
 316         new_table = { (rewrite_file_name(file_name, self.config, True), file_name)
 317                       for file_name in filelist }
 318
 319         # Setup the file table.
 320         filetable = GLFileTable(filelist)
 321         filetable.new_files = sorted(new_table, key=lambda pair: pair[0])
 322
 323         # Create directories.
 324         directories = sorted({ joinpath(self.testdir, os.path.dirname(pair[0]))
 325                                for pair in filetable.new_files })
 326         for directory in directories:
 327             if not os.path.isdir(directory):
 328                 os.makedirs(directory)
 329
 330         # Copy files or make symbolic links or hard links.
 331         for (dest, src) in filetable.new_files:
 332             destpath = joinpath(self.testdir, dest)
 333             if src.startswith('tests=lib/'):
 334                 src = substart('tests=lib/', 'lib/', src)
 335             lookedup, flag = self.filesystem.lookup(src)
 336             if os.path.isfile(destpath):
 337                 os.remove(destpath)
 338             if flag:
 339                 copyfile(lookedup, destpath)
 340                 ensure_writable(destpath)
 341             else:  # if not flag
 342                 if self.filesystem.shouldLink(src, lookedup) == CopyAction.Symlink:
 343                     link_relative(lookedup, destpath)
 344                 elif self.filesystem.shouldLink(src, lookedup) == CopyAction.Hardlink:
 345                     hardlink(lookedup, destpath)
 346                 else:
 347                     copyfile(lookedup, destpath)
 348                     ensure_writable(destpath)
 349
 350         # Create $sourcebase/Makefile.am.
 351         for_test = True
 352         directory = joinpath(self.testdir, sourcebase)
 353         if not os.path.isdir(directory):
 354             os.mkdir(directory)
 355         destfile = joinpath(directory, 'Makefile.am')
 356         if single_configure:
 357             emit = self.emitter.lib_Makefile_am(destfile, main_modules,
 358                                                 moduletable, self.makefiletable, '', for_test)
 359         else:  # if not single_configure
 360             emit = self.emitter.lib_Makefile_am(destfile, modules,
 361                                                 moduletable, self.makefiletable, '', for_test)
 362         with open(destfile, mode='w', newline='\n', encoding='utf-8') as file:
 363             file.write(emit)
 364
 365         # Create $m4base/Makefile.am.
 366         directory = joinpath(self.testdir, m4base)
 367         if not os.path.isdir(directory):
 368             os.mkdir(directory)
 369         destfile = joinpath(directory, 'Makefile.am')
 370         emit = '## Process this file with automake to produce Makefile.in.\n\n'
 371         emit += 'EXTRA_DIST =\n'
 372         for file in filetable.all_files:
 373             if file.startswith('m4/'):
 374                 file = substart('m4/', '', file)
 375                 emit += 'EXTRA_DIST += %s\n' % file
 376         with open(destfile, mode='w', newline='\n', encoding='utf-8') as file:
 377             file.write(emit)
 378
 379         subdirs = [sourcebase, m4base]
 380         subdirs_with_configure_ac = []
 381
 382         inctests = self.config.checkInclTestCategory(TESTS['tests'])
 383         if inctests:
 384             directory = joinpath(self.testdir, testsbase)
 385             if not os.path.isdir(directory):
 386                 os.mkdir(directory)
 387             if single_configure:
 388                 # Create $testsbase/Makefile.am.
 389                 destfile = joinpath(directory, 'Makefile.am')
 390                 witness_macro = '%stests_WITNESS' % macro_prefix
 391                 emit = self.emitter.tests_Makefile_am(destfile, tests_modules, moduletable,
 392                                                       self.makefiletable, witness_macro, for_test)
 393                 with open(destfile, mode='w', newline='\n', encoding='utf-8') as file:
 394                     file.write(emit)
 395             else:  # if not single_configure
 396                 # Create $testsbase/Makefile.am.
 397                 destfile = joinpath(directory, 'Makefile.am')
 398                 libtests = False
 399                 self.config.setLibtests(False)
 400                 emit = self.emitter.tests_Makefile_am(destfile, modules, moduletable,
 401                                                       self.makefiletable, '', for_test)
 402                 with open(destfile, mode='w', newline='\n', encoding='utf-8') as file:
 403                     file.write(emit)
 404                 # Viewed from the $testsbase subdirectory, $auxdir is different.
 405                 emit = ''
 406                 saved_auxdir = auxdir
 407                 auxdir = os.path.normpath(joinpath(relinverse(testsbase), auxdir))
 408                 self.config.setAuxDir(auxdir)
 409                 # Create $testsbase/configure.ac.
 410                 emit += '# Process this file with autoconf '
 411                 emit += 'to produce a configure script.\n'
 412                 emit += 'AC_INIT([dummy], [0])\n'
 413                 emit += 'AC_CONFIG_AUX_DIR([%s])\n' % auxdir
 414                 emit += 'AM_INIT_AUTOMAKE\n\n'
 415                 emit += 'AC_CONFIG_HEADERS([config.h])\n\n'
 416                 emit += 'AC_PROG_CC\n'
 417                 emit += 'AC_PROG_INSTALL\n'
 418                 emit += 'AC_PROG_MAKE_SET\n'
 419                 emit += self.emitter.preEarlyMacros(False, '', modules)
 420                 snippets = []
 421                 for module in modules:
 422                     if module.name in ['gnumakefile', 'maintainer-makefile']:
 423                         # These are meant to be used only in the top-level directory.
 424                         pass
 425                     # if module.name not in ['gnumakefile', 'maintainer-makefile']
 426                     else:
 427                         snippet = module.getAutoconfEarlySnippet()
 428                         lines = [ line
 429                                   for line in snippet.split('\n')
 430                                   if line.strip() ]
 431                         snippet = lines_to_multiline(lines)
 432                         pattern = re.compile(r'AC_REQUIRE\(\[([^()]*)]\)', re.M)
 433                         snippet = pattern.sub(r'\1', snippet)
 434                         snippet = snippet.strip()
 435                         snippets.append(snippet)
 436                 snippets = [ snippet
 437                              for snippet in snippets
 438                              if snippet.strip()]
 439                 emit += lines_to_multiline(snippets)
 440                 if libtool:
 441                     emit += 'LT_INIT([win32-dll])\n'
 442                     emit += 'LT_LANG([C++])\n'
 443                     emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [true])\n'
 444                     emit += 'gl_cond_libtool=true\n'
 445                 else:  # if not libtool
 446                     emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [false])\n'
 447                     emit += 'gl_cond_libtool=false\n'
 448                     emit += 'gl_libdeps=\n'
 449                     emit += 'gl_ltlibdeps=\n'
 450                 # Wrap the set of autoconf snippets into an autoconf macro that is then
 451                 # invoked. This is needed because autoconf does not support AC_REQUIRE
 452                 # at the top level:
 453                 #   error: AC_REQUIRE(gt_CSHARPCOMP): cannot be used outside of an
 454                 #     AC_DEFUN'd macro
 455                 # but we want the AC_REQUIRE to have its normal meaning (provide one
 456                 # expansion of the required macro before the current point, and only
 457                 # one expansion total).
 458                 emit += 'AC_DEFUN([gl_INIT], [\n'
 459                 replace_auxdir = True
 460                 emit += "gl_m4_base='../%s'\n" % m4base
 461                 emit += self.emitter.initmacro_start(macro_prefix, True)
 462                 # We don't have explicit ordering constraints between the various
 463                 # autoconf snippets. It's cleanest to put those of the library before
 464                 # those of the tests.
 465                 emit += self.emitter.shellvars_init(True, f'../{sourcebase}')
 466                 emit += self.emitter.autoconfSnippets(modules, modules, moduletable,
 467                                                       lambda module: module.isNonTests(),
 468                                                       False, False, False, replace_auxdir)
 469                 emit += self.emitter.shellvars_init(True, '.')
 470                 emit += self.emitter.autoconfSnippets(modules, modules, moduletable,
 471                                                       lambda module: module.isTests(),
 472                                                       False, False, False, replace_auxdir)
 473                 emit += self.emitter.initmacro_end(macro_prefix, True)
 474                 # _LIBDEPS and _LTLIBDEPS variables are not needed if this library is
 475                 # created using libtool, because libtool already handles the
 476                 # dependencies.
 477                 if not libtool:
 478                     libname_upper = libname.upper().replace('-', '_')
 479                     emit += '  %s_LIBDEPS="$gl_libdeps"\n' % libname_upper
 480                     emit += '  AC_SUBST([%s_LIBDEPS])\n' % libname_upper
 481                     emit += '  %s_LTLIBDEPS="$gl_ltlibdeps"\n' % libname_upper
 482                     emit += '  AC_SUBST([%s_LTLIBDEPS])\n' % libname_upper
 483                 emit += '])\n'
 484                 # FIXME use $sourcebase or $testsbase?
 485                 emit += self.emitter.initmacro_done(macro_prefix, sourcebase)
 486                 emit += '\ngl_INIT\n\n'
 487                 # Usually $testsbase/config.h will be a superset of config.h. Verify
 488                 # this by "merging" config.h into $testsbase/config.h; look out for gcc
 489                 # warnings.
 490                 emit += 'AH_TOP([#include \"../config.h\"])\n\n'
 491                 emit += 'AC_CONFIG_FILES([Makefile])\n'
 492                 emit += 'AC_OUTPUT\n'
 493                 path = joinpath(self.testdir, testsbase, 'configure.ac')
 494                 with open(path, mode='w', newline='\n', encoding='utf-8') as file:
 495                     file.write(emit)
 496
 497                 # Restore changed variables.
 498                 self.config.setAuxDir(saved_auxdir)
 499                 auxdir = self.config['auxdir']
 500                 subdirs_with_configure_ac.append(testsbase)
 501
 502             subdirs.append(testsbase)
 503
 504         # Create Makefile.am.
 505         emit = '## Process this file with automake to produce Makefile.in.\n\n'
 506         emit += 'AUTOMAKE_OPTIONS = 1.14 foreign\n\n'
 507         emit += 'SUBDIRS = %s\n\n' % ' '.join(subdirs)
 508         emit += 'ACLOCAL_AMFLAGS = -I %s\n' % m4base
 509         path = joinpath(self.testdir, 'Makefile.am')
 510         with open(path, mode='w', newline='\n', encoding='utf-8') as file:
 511             file.write(emit)
 512
 513         # Create configure.ac
 514         emit = '# Process this file with autoconf '
 515         emit += 'to produce a configure script.\n'
 516         emit += 'AC_INIT([dummy], [0])\n'
 517         if auxdir != '.':
 518             emit += 'AC_CONFIG_AUX_DIR([%s])\n' % auxdir
 519         emit += 'AM_INIT_AUTOMAKE\n\n'
 520         emit += 'AC_CONFIG_HEADERS([config.h])\n\n'
 521         emit += 'AC_PROG_CC\n'
 522         emit += 'AC_PROG_INSTALL\n'
 523         emit += 'AC_PROG_MAKE_SET\n\n'
 524         emit += '# For autobuild.\n'
 525         emit += 'AC_CANONICAL_BUILD\n'
 526         emit += 'AC_CANONICAL_HOST\n\n'
 527         emit += 'm4_pattern_forbid([^gl_[A-Z]])dnl the gnulib macro namespace\n'
 528         emit += 'm4_pattern_allow([^gl_ES$])dnl a valid locale name\n'
 529         emit += 'm4_pattern_allow([^gl_LIBOBJS$])dnl a variable\n'
 530         emit += 'm4_pattern_allow([^gl_LTLIBOBJS$])dnl a variable\n'
 531         emit += self.emitter.preEarlyMacros(False, '', modules)
 532         snippets = []
 533         for module in final_modules:
 534             if single_configure:
 535                 solution = True
 536             else:  # if not single_configure
 537                 solution = module.isNonTests()
 538             if solution:
 539                 snippet = module.getAutoconfEarlySnippet()
 540                 lines = [ line
 541                           for line in snippet.split('\n')
 542                           if line.strip() ]
 543                 snippet = lines_to_multiline(lines)
 544                 pattern = re.compile(r'AC_REQUIRE\(\[([^()]*)]\)', re.M)
 545                 snippet = pattern.sub(r'\1', snippet)
 546                 snippet = snippet.strip()
 547                 snippets.append(snippet)
 548         snippets = [ snippet
 549                      for snippet in snippets
 550                      if snippet.strip() ]
 551         emit += lines_to_multiline(snippets)
 552         if libtool:
 553             emit += 'LT_INIT([win32-dll])\n'
 554             emit += 'LT_LANG([C++])\n'
 555             emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [true])\n'
 556             emit += 'gl_cond_libtool=true\n'
 557         else:  # if not libtool
 558             emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [false])\n'
 559             emit += 'gl_cond_libtool=false\n'
 560             emit += 'gl_libdeps=\n'
 561             emit += 'gl_ltlibdeps=\n'
 562         # Wrap the set of autoconf snippets into an autoconf macro that is then
 563         # invoked. This is needed because autoconf does not support AC_REQUIRE
 564         # at the top level:
 565         #   error: AC_REQUIRE(gt_CSHARPCOMP): cannot be used outside of an
 566         #     AC_DEFUN'd macro
 567         # but we want the AC_REQUIRE to have its normal meaning (provide one
 568         # expansion of the required macro before the current point, and only one
 569         # expansion total).
 570         emit += 'AC_DEFUN([gl_INIT], [\n'
 571         if auxdir != 'build-aux':
 572             replace_auxdir = True
 573         else:  # auxdir == 'build-aux'
 574             replace_auxdir = False
 575         emit += 'gl_m4_base=\'%s\'\n' % m4base
 576         emit += self.emitter.initmacro_start(macro_prefix, False)
 577         emit += self.emitter.shellvars_init(False, sourcebase)
 578         if single_configure:
 579             emit += self.emitter.autoconfSnippets(main_modules, main_modules, moduletable,
 580                                                   lambda module: True,
 581                                                   True, False, False, replace_auxdir)
 582         else:  # if not single_configure
 583             emit += self.emitter.autoconfSnippets(modules, modules, moduletable,
 584                                                   lambda module: module.isNonTests(),
 585                                                   True, False, False, replace_auxdir)
 586         emit += self.emitter.initmacro_end(macro_prefix, False)
 587         if single_configure:
 588             emit += '  gltests_libdeps=\n'
 589             emit += '  gltests_ltlibdeps=\n'
 590             emit += self.emitter.initmacro_start('%stests' % macro_prefix, True)
 591             emit += self.emitter.shellvars_init(True, testsbase)
 592             # Define a tests witness macro.
 593             emit += '  %stests_WITNESS=IN_GNULIB_TESTS\n' % macro_prefix
 594             emit += '  AC_SUBST([%stests_WITNESS])\n' % macro_prefix
 595             emit += '  gl_module_indicator_condition=$%stests_WITNESS\n' % macro_prefix
 596             emit += '  m4_pushdef([gl_MODULE_INDICATOR_CONDITION], '
 597             emit += '[$gl_module_indicator_condition])\n'
 598             snippets = self.emitter.autoconfSnippets(tests_modules, main_modules + tests_modules,
 599                                                      moduletable, lambda module: True,
 600                                                      True, False, False, replace_auxdir)
 601             emit += snippets
 602             emit += '  m4_popdef([gl_MODULE_INDICATOR_CONDITION])\n'
 603             emit += self.emitter.initmacro_end('%stests' % macro_prefix, True)
 604         # _LIBDEPS and _LTLIBDEPS variables are not needed if this library is
 605         # created using libtool, because libtool already handles the dependencies.
 606         if not libtool:
 607             libname_upper = libname.upper().replace('-', '_')
 608             emit += '  %s_LIBDEPS="$gl_libdeps"\n' % libname_upper
 609             emit += '  AC_SUBST([%s_LIBDEPS])\n' % libname_upper
 610             emit += '  %s_LTLIBDEPS="$gl_ltlibdeps"\n' % libname_upper
 611             emit += '  AC_SUBST([%s_LTLIBDEPS])\n' % libname_upper
 612         if single_configure and libtests:
 613             emit += '  LIBTESTS_LIBDEPS="$gltests_libdeps"\n'
 614             emit += '  AC_SUBST([LIBTESTS_LIBDEPS])\n'
 615         emit += '])\n'
 616         emit += self.emitter.initmacro_done(macro_prefix, sourcebase)
 617         if single_configure:
 618             emit += self.emitter.initmacro_done('%stests' % macro_prefix, testsbase)
 619         emit += '\ngl_INIT\n\n'
 620         if subdirs_with_configure_ac:
 621             if single_configure:
 622                 emit += 'AC_CONFIG_SUBDIRS([%s])\n' % ' '.join(subdirs_with_configure_ac[:-1])
 623             else:  # if not single_configure
 624                 emit += 'AC_CONFIG_SUBDIRS([%s])\n' % ' '.join(subdirs_with_configure_ac)
 625         makefiles = ['Makefile']
 626         for directory in subdirs:
 627             # For subdirs that have a configure.ac by their own, it's the subdir's
 628             # configure.ac which creates the subdir's Makefile.am, not this one.
 629             makefiles.append(joinpath(directory, 'Makefile'))
 630         if not single_configure:
 631             makefiles = makefiles[:-1]
 632         emit += 'AC_CONFIG_FILES([%s])\n' % ' '.join(makefiles)
 633         emit += 'AC_OUTPUT\n'
 634         path = joinpath(self.testdir, 'configure.ac')
 635         with open(path, mode='w', newline='\n', encoding='utf-8') as file:
 636             file.write(emit)
 637
 638         # Create autogenerated files.
 639         # Do not use "${AUTORECONF} --force --install", because it may invoke
 640         # autopoint, which brings in older versions of some of our .m4 files.
 641         force_output()
 642         os.chdir(self.testdir)
 643         # gettext
 644         if os.path.isfile(joinpath(m4base, 'gettext.m4')):
 645             args = [UTILS['autopoint'], '--force']
 646             execute(args, verbose)
 647             for src in os.listdir(m4base):
 648                 src = joinpath(m4base, src)
 649                 if src.endswith('.m4~'):
 650                     dest = src[:-1]
 651                     if os.path.isfile(dest):
 652                         os.remove(dest)
 653                     movefile(src, dest)
 654         # libtoolize
 655         if libtool:
 656             args = [UTILS['libtoolize'], '--copy']
 657             execute(args, verbose)
 658         # aclocal
 659         args = [UTILS['aclocal'], '-I', m4base]
 660         execute(args, verbose)
 661         if not os.path.isdir('build-aux'):
 662             print('executing mkdir build-aux')
 663             os.mkdir('build-aux')
 664         # autoconf
 665         args = [UTILS['autoconf']]
 666         execute(args, verbose)
 667         # autoheader
 668         args = [UTILS['autoheader']]
 669         execute(args, verbose)
 670         # Explicit 'touch config.h.in': see <https://savannah.gnu.org/support/index.php?109406>.
 671         print('executing touch config.h.in')
 672         Path('config.h.in').touch()
 673         # automake
 674         args = [UTILS['automake'], '--add-missing', '--copy']
 675         execute(args, verbose)
 676         rmtree('autom4te.cache')
 677         os.chdir(DIRS['cwd'])
 678         if inctests and not single_configure:
 679             # Do not use "${AUTORECONF} --force --install", because it may invoke
 680             # autopoint, which brings in older versions of some of our .m4 files.
 681             os.chdir(joinpath(self.testdir, testsbase))
 682             # gettext
 683             if os.path.isfile(joinpath(m4base, 'gettext.m4')):
 684                 args = [UTILS['autopoint'], '--force']
 685                 execute(args, verbose)
 686                 for src in os.listdir(m4base):
 687                     src = joinpath(m4base, src)
 688                     if src.endswith('.m4~'):
 689                         dest = src[:-1]
 690                         if os.path.isfile(dest):
 691                             os.remove(dest)
 692                         movefile(src, dest)
 693             # aclocal
 694             args = [UTILS['aclocal'], '-I', joinpath('..', m4base)]
 695             execute(args, verbose)
 696             if not os.path.isdir(joinpath('../build-aux')):
 697                 print('executing mkdir ../build-aux')
 698                 os.mkdir('../build-aux')
 699             # autoconf
 700             args = [UTILS['autoconf']]
 701             execute(args, verbose)
 702             # autoheader
 703             args = [UTILS['autoheader']]
 704             execute(args, verbose)
 705             # Explicit 'touch config.h.in': see <https://savannah.gnu.org/support/index.php?109406>.
 706             print('executing touch config.h.in')
 707             Path('config.h.in').touch()
 708             # automake
 709             args = [UTILS['automake'], '--add-missing', '--copy']
 710             execute(args, verbose)
 711             rmtree('autom4te.cache')
 712             os.chdir(DIRS['cwd'])
 713
 714         # Need to run configure and make once, to create built files that are to be
 715         # distributed (such as parse-datetime.c).
 716         path = joinpath(self.testdir, sourcebase, 'Makefile.am')
 717         with open(path, mode='r', newline='\n', encoding='utf-8') as file:
 718             snippet = file.read()
 719         snippet = combine_lines(snippet)
 720
 721         # Extract the value of "CLEANFILES += ..." and "MOSTLYCLEANFILES += ...".
 722         regex_find = []
 723         pattern = re.compile(r'^CLEANFILES[\t ]*\+=(.*)$', re.M)
 724         regex_find += pattern.findall(snippet)
 725         pattern = re.compile(r'^MOSTLYCLEANFILES[\t ]*\+=(.*)$', re.M)
 726         regex_find += pattern.findall(snippet)
 727         regex_find = [ line.strip()
 728                        for line in regex_find
 729                        if line.strip() ]
 730         cleaned_files = []
 731         for part in regex_find:
 732             cleaned_files += \
 733                 [ line.strip()
 734                   for line in part.split(' ')
 735                   if line.strip() ]
 736
 737         # Extract the value of "BUILT_SOURCES += ...". Remove variable references
 738         # such $(FOO_H) because they don't refer to distributed files.
 739         regex_find = []
 740         pattern = re.compile(r'^BUILT_SOURCES[\t ]*\+=(.*)$', re.M)
 741         regex_find += pattern.findall(snippet)
 742         regex_find = [ line.strip()
 743                        for line in regex_find
 744                        if line.strip()]
 745         built_sources = []
 746         for part in regex_find:
 747             built_sources += \
 748                 [ line.strip()
 749                   for line in part.split(' ')
 750                   if line.strip()]
 751         built_sources = [ line
 752                           for line in built_sources
 753                           if not bool(re.compile(r'[$]\([A-Za-z0-9_]*\)$').findall(line)) ]
 754         distributed_built_sources = [ file
 755                                       for file in built_sources
 756                                       if file not in cleaned_files ]
 757
 758         tests_distributed_built_sources = []
 759         if inctests:
 760             # Likewise for built files in the $testsbase directory.
 761             path = joinpath(self.testdir, testsbase, 'Makefile.am')
 762             with open(path, mode='r', newline='\n', encoding='utf-8') as file:
 763                 snippet = file.read()
 764             snippet = combine_lines(snippet)
 765
 766             # Extract the value of "CLEANFILES += ..." and "MOSTLYCLEANFILES += ...".
 767             regex_find = []
 768             pattern = re.compile(r'^CLEANFILES[\t ]*\+=(.*)$', re.M)
 769             regex_find += pattern.findall(snippet)
 770             pattern = re.compile(r'^MOSTLYCLEANFILES[\t ]*\+=(.*)$', re.M)
 771             regex_find += pattern.findall(snippet)
 772             regex_find = [ line.strip()
 773                            for line in regex_find
 774                            if line.strip() ]
 775             tests_cleaned_files = []
 776             for part in regex_find:
 777                 tests_cleaned_files += \
 778                     [ line.strip()
 779                       for line in part.split(' ')
 780                       if line.strip() ]
 781
 782             # Extract the value of "BUILT_SOURCES += ...". Remove variable references
 783             # such $(FOO_H) because they don't refer to distributed files.
 784             regex_find = []
 785             pattern = re.compile(r'^BUILT_SOURCES[\t ]*\+=(.*)$', re.M)
 786             regex_find += pattern.findall(snippet)
 787             regex_find = [ line.strip()
 788                            for line in regex_find
 789                            if line.strip() ]
 790             tests_built_sources = []
 791             for part in regex_find:
 792                 tests_built_sources += \
 793                     [ line.strip()
 794                       for line in part.split(' ')
 795                       if line.strip() ]
 796             tests_built_sources = [ line
 797                                     for line in tests_built_sources
 798                                     if not bool(re.compile(r'[$]\([A-Za-z0-9_]*\)$').findall(line)) ]
 799             tests_distributed_built_sources = [ file
 800                                                 for file in tests_built_sources
 801                                                 if file not in tests_cleaned_files]
 802
 803         os.chdir(self.testdir)
 804         if distributed_built_sources or tests_distributed_built_sources:
 805             force_output()
 806             sp.call('./configure')
 807             if distributed_built_sources:
 808                 os.chdir(sourcebase)
 809                 with open('Makefile', mode='a', newline='\n', encoding='utf-8') as file:
 810                     file.write('built_sources: $(BUILT_SOURCES)\n')
 811                 args = [UTILS['make'],
 812                         'AUTOCONF=%s' % UTILS['autoconf'],
 813                         'AUTOHEADER=%s' % UTILS['autoheader'],
 814                         'ACLOCAL=%s' % UTILS['aclocal'],
 815                         'AUTOMAKE=%s' % UTILS['automake'],
 816                         'AUTORECONF=%s' % UTILS['autoreconf'],
 817                         'built_sources']
 818                 sp.call(args)
 819                 os.chdir('..')
 820             if tests_distributed_built_sources:
 821                 os.chdir(testsbase)
 822                 with open('Makefile', mode='a', newline='\n', encoding='utf-8') as file:
 823                     file.write('built_sources: $(BUILT_SOURCES)\n')
 824                 args = [UTILS['make'],
 825                         'AUTOCONF=%s' % UTILS['autoconf'],
 826                         'AUTOHEADER=%s' % UTILS['autoheader'],
 827                         'ACLOCAL=%s' % UTILS['aclocal'],
 828                         'AUTOMAKE=%s' % UTILS['automake'],
 829                         'AUTORECONF=%s' % UTILS['autoreconf'],
 830                         'built_sources']
 831                 sp.call(args)
 832                 os.chdir('..')
 833             args = [UTILS['make'],
 834                     'AUTOCONF=%s' % UTILS['autoconf'],
 835                     'AUTOHEADER=%s' % UTILS['autoheader'],
 836                     'ACLOCAL=%s' % UTILS['aclocal'],
 837                     'AUTOMAKE=%s' % UTILS['automake'],
 838                     'AUTORECONF=%s' % UTILS['autoreconf'],
 839                     'AUTOPOINT=%s' % UTILS['autopoint'],
 840                     'LIBTOOLIZE=%s' % UTILS['libtoolize'],
 841                     'distclean']
 842             sp.call(args)
 843         if os.path.isfile(joinpath('build-aux', 'test-driver')):
 844             _patch_test_driver()
 845         os.chdir(DIRS['cwd'])
 846
 847
 848 #===============================================================================
 849 # Define GLMegaTestDir class
 850 #===============================================================================
 851 class GLMegaTestDir:
 852     '''GLMegaTestDir class is used to create a mega scratch package with the
 853     given modules one by one and all together.'''
 854
 855     config: GLConfig
 856     megatestdir: str
 857     modulesystem: GLModuleSystem
 858
 859     def __init__(self, config: GLConfig, megatestdir: str) -> None:
 860         '''Create new GLTestDir instance.'''
 861         if type(config) is not GLConfig:
 862             raise TypeError('config must be a GLConfig, not %s'
 863                             % type(config).__name__)
 864         if type(megatestdir) is not str:
 865             raise TypeError('megatestdir must be a string, not %s'
 866                             % type(megatestdir).__name__)
 867         self.config = config
 868         self.megatestdir = os.path.normpath(megatestdir)
 869         # Don't overwrite the directory.
 870         if os.path.exists(self.megatestdir):
 871             raise GLError(22, self.megatestdir)
 872         # Try to create directory.
 873         try:
 874             os.mkdir(self.megatestdir)
 875         except Exception as exc:
 876             raise GLError(19, self.megatestdir) from exc
 877         self.modulesystem = GLModuleSystem(self.config)
 878
 879     def execute(self) -> None:
 880         '''Create a mega scratch package with the given modules one by one
 881         and all together.'''
 882         auxdir = self.config['auxdir']
 883         verbose = self.config['verbosity']
 884
 885         megasubdirs = []
 886         modules = [ self.modulesystem.find(m)
 887                     for m in self.config['modules'] ]
 888         if not modules:
 889             modules = self.modulesystem.list()
 890             modules = [ self.modulesystem.find(m)
 891                         for m in modules ]
 892         # Preserve ordering from the command-line, but remove duplicates.
 893         # This allows control over the SUBDIRS variable in the top-level Makefile.am.
 894         module_set = set(modules)
 895         modules = [ module
 896                     for module in modules
 897                     if module in module_set ]
 898
 899         # First, all modules one by one.
 900         for module in modules:
 901             self.config.setModules([module.name])
 902             GLTestDir(self.config, joinpath(self.megatestdir, module.name)).execute()
 903             megasubdirs.append(module.name)
 904
 905         # Then, all modules all together.
 906         # Except config-h, which breaks all modules which use HAVE_CONFIG_H.
 907         modules = [ module
 908                     for module in modules
 909                     if module.name != 'config-h' ]
 910         self.config.setModules([ module.name
 911                                  for module in modules ])
 912         GLTestDir(self.config, joinpath(self.megatestdir, 'ALL')).execute()
 913         megasubdirs.append('ALL')
 914
 915         # Create autobuild.
 916         emit = ''
 917         repdict = dict()
 918         repdict['Jan'] = repdict['January'] = '01'
 919         repdict['Feb'] = repdict['February'] = '02'
 920         repdict['Mar'] = repdict['March'] = '03'
 921         repdict['Apr'] = repdict['April'] = '04'
 922         repdict['May'] = repdict['May'] = '05'
 923         repdict['Jun'] = repdict['June'] = '06'
 924         repdict['Jul'] = repdict['July'] = '07'
 925         repdict['Aug'] = repdict['August'] = '08'
 926         repdict['Sep'] = repdict['September'] = '09'
 927         repdict['Oct'] = repdict['October'] = '10'
 928         repdict['Nov'] = repdict['November'] = '11'
 929         repdict['Dec'] = repdict['December'] = '12'
 930         vc_witness = joinpath(DIRS['root'], '.git', 'refs', 'heads', 'master')
 931         if not os.path.isfile(vc_witness):
 932             vc_witness = joinpath(DIRS['root'], 'ChangeLog')
 933         mdate_sh = joinpath(DIRS['root'], 'build-aux', 'mdate-sh')
 934         args = ['sh', mdate_sh, vc_witness]
 935         cvsdate = sp.check_output(args).decode('UTF-8').strip()
 936         for key in repdict:
 937             if len(key) > 3:
 938                 cvsdate = cvsdate.replace(key, repdict[key])
 939         for key in repdict:
 940             cvsdate = cvsdate.replace(key, repdict[key])
 941         cvsdate = ''.join([ date
 942                             for date in cvsdate.split(' ')
 943                             if date.strip() ])
 944         cvsdate = '%s%s%s' % (cvsdate[4:], cvsdate[2:4], cvsdate[:2])
 945         emit += '#!/bin/sh\n'
 946         emit += 'CVSDATE=%s\n' % cvsdate
 947         emit += ': ${MAKE=make}\n'
 948         emit += 'test -d logs || mkdir logs\n'
 949         emit += 'for module in %s; do\n' % ' '.join(megasubdirs)
 950         emit += '  echo "Working on module $module..."\n'
 951         emit += '  safemodule=`echo $module | sed -e \'s|/|-|g\'`\n'
 952         emit += '  (echo "To: gnulib@autobuild.josefsson.org"\n'
 953         emit += '   echo\n'
 954         emit += '   set -x\n'
 955         emit += '   : autobuild project... $module\n'
 956         emit += '   : autobuild revision... cvs-$CVSDATE-000000\n'
 957         emit += '   : autobuild timestamp... `date "+%Y%m%d-%H%M%S"`\n'
 958         emit += '   : autobuild hostname... `hostname`\n'
 959         emit += '   cd $module && ./configure $CONFIGURE_OPTIONS && $MAKE'
 960         emit += ' && $MAKE check && $MAKE distclean\n'
 961         emit += '   echo rc=$?\n'
 962         emit += '  ) 2>&1 | { if test -n "$AUTOBUILD_SUBST"; then '
 963         emit += 'sed -e "$AUTOBUILD_SUBST"; else cat; fi; } > logs/$safemodule\n'
 964         emit += 'done\n'
 965         path = joinpath(self.megatestdir, 'do-autobuild')
 966         with open(path, mode='w', newline='\n', encoding='utf-8') as file:
 967             file.write(emit)
 968
 969         # Create Makefile.am.
 970         emit = '## Process this file with automake to produce Makefile.in.\n\n'
 971         emit += 'AUTOMAKE_OPTIONS = 1.14 foreign\n\n'
 972         emit += 'SUBDIRS = %s\n\n' % ' '.join(megasubdirs)
 973         emit += 'EXTRA_DIST = do-autobuild\n'
 974         path = joinpath(self.megatestdir, 'Makefile.am')
 975         with open(path, mode='w', newline='\n', encoding='utf-8') as file:
 976             file.write(emit)
 977
 978         emit = '# Process this file with autoconf '
 979         emit += 'to produce a configure script.\n'
 980         emit += 'AC_INIT([dummy], [0])\n'
 981         if auxdir != '.':
 982             emit += 'AC_CONFIG_AUX_DIR([%s])\n' % auxdir
 983         emit += 'AM_INIT_AUTOMAKE\n\n'
 984         emit += 'AC_PROG_MAKE_SET\n\n'
 985         emit += 'AC_CONFIG_SUBDIRS([%s])\n' % ' '.join(megasubdirs)
 986         emit += 'AC_CONFIG_FILES([Makefile])\n'
 987         emit += 'AC_OUTPUT\n'
 988         path = joinpath(self.megatestdir, 'configure.ac')
 989         with open(path, mode='w', newline='\n', encoding='utf-8') as file:
 990             file.write(emit)
 991
 992         # Create autogenerated files.
 993         force_output()
 994         os.chdir(self.megatestdir)
 995         args = [UTILS['aclocal']]
 996         execute(args, verbose)
 997         try:  # Try to make a directory
 998             if not os.path.isdir('build-aux'):
 999                 print('executing mkdir build-aux')
1000                 os.mkdir('build-aux')
1001         except Exception:
1002             pass
1003         args = [UTILS['autoconf']]
1004         execute(args, verbose)
1005         args = [UTILS['automake'], '--add-missing', '--copy']
1006         execute(args, verbose)
1007         rmtree('autom4te.cache')
1008         if os.path.isfile(joinpath('build-aux', 'test-driver')):
1009             _patch_test_driver()
1010         os.chdir(DIRS['cwd'])