pygnulib/GLTestDir.py

   1 #!/usr/bin/python
   2 # encoding: UTF-8
   3
   4 #===============================================================================
   5 # Define global imports
   6 #===============================================================================
   7 import os
   8 import re
   9 import sys
  10 import locale
  11 import codecs
  12 import shutil
  13 import filecmp
  14 import subprocess as sp
  15 from . import constants
  16 from .GLError import GLError
  17 from .GLConfig import GLConfig
  18 from .GLModuleSystem import GLModule
  19 from .GLModuleSystem import GLModuleTable
  20 from .GLModuleSystem import GLModuleSystem
  21 from .GLFileSystem import GLFileSystem
  22 from .GLFileSystem import GLFileAssistant
  23 from .GLMakefileTable import GLMakefileTable
  24 from .GLEmiter import GLEmiter
  25
  26
  27 #===============================================================================
  28 # Define module information
  29 #===============================================================================
  30 __author__ = constants.__author__
  31 __license__ = constants.__license__
  32 __copyright__ = constants.__copyright__
  33
  34
  35 #===============================================================================
  36 # Define global constants
  37 #===============================================================================
  38 PYTHON3 = constants.PYTHON3
  39 NoneType = type(None)
  40 APP = constants.APP
  41 DIRS = constants.DIRS
  42 ENCS = constants.ENCS
  43 UTILS = constants.UTILS
  44 MODES = constants.MODES
  45 TESTS = constants.TESTS
  46 compiler = constants.compiler
  47 joinpath = constants.joinpath
  48 cleaner = constants.cleaner
  49 relpath = constants.relativize
  50 string = constants.string
  51 isabs = os.path.isabs
  52 isdir = os.path.isdir
  53 isfile = os.path.isfile
  54 normpath = os.path.normpath
  55
  56
  57 #===============================================================================
  58 # Define GLTestDir class
  59 #===============================================================================
  60 class GLTestDir(object):
  61     '''GLTestDir class is used to create a scratch package with the given
  62     list of the modules.'''
  63
  64     def __init__(self, config, testdir):
  65         '''GLTestDir.__init__(config, testdir) -> GLTestDir
  66
  67         Create new GLTestDir instance.'''
  68         if type(config) is not GLConfig:
  69             raise(TypeError('config must be a GLConfig, not %s' %
  70                             type(config).__name__))
  71         if type(testdir) is bytes or type(testdir) is string:
  72             if type(testdir) is bytes:
  73                 testdir = testdir.decode(ENCS['default'])
  74         self.config = config
  75         self.testdir = os.path.normpath(testdir)
  76         if not os.path.exists(self.testdir):
  77             try:  # Try to create directory
  78                 os.mkdir(self.testdir)
  79             except Exception as error:
  80                 raise(GLError(19, self.testdir))
  81         self.emiter = GLEmiter(self.config)
  82         self.filesystem = GLFileSystem(self.config)
  83         self.modulesystem = GLModuleSystem(self.config)
  84         self.moduletable = GLModuleTable(self.config)
  85         self.assistant = GLFileAssistant(self.config)
  86         self.makefiletable = GLMakefileTable(self.config)
  87
  88         # Subdirectory names.
  89         self.config.setSourceBase('gllib')
  90         self.config.setM4Base('glm4')
  91         self.config.setDocBase('gldoc')
  92         self.config.setTestsBase('gltests')
  93         self.config.setMacroPrefix('gl')
  94         self.config.resetPoBase()
  95         self.config.resetPoDomain()
  96         self.config.resetWitnessCMacro()
  97         self.config.resetVCFiles()
  98
  99     def rewrite_files(self, files):
 100         '''GLTestDir.rewrite_files(files)
 101
 102         Replace auxdir, docbase, sourcebase, m4base and testsbase from default
 103         to their version from config.'''
 104         if type(files) is not list:
 105             raise(TypeError(
 106                 'files argument must has list type, not %s' % type(files).__name__))
 107         files = \
 108             [  # Begin to convert bytes to string
 109                 file.decode(ENCS['default']) \
 110                 if type(file) is bytes else file \
 111                 for file in files
 112             ]  # Finish to convert bytes to string
 113         for file in files:
 114             if type(file) is not string:
 115                 raise(TypeError('each file must be a string instance'))
 116         files = sorted(set(files))
 117         auxdir = self.config['auxdir']
 118         docbase = self.config['docbase']
 119         sourcebase = self.config['sourcebase']
 120         m4base = self.config['m4base']
 121         testsbase = self.config['testsbase']
 122         result = list()
 123         for file in files:
 124             if file.startswith('build-aux/'):
 125                 path = constants.substart('build-aux/', '%s/' % auxdir, file)
 126             elif file.startswith('doc/'):
 127                 path = constants.substart('doc/', '%s/' % docbase, file)
 128             elif file.startswith('lib/'):
 129                 path = constants.substart('lib/', '%s/' % sourcebase, file)
 130             elif file.startswith('m4/'):
 131                 path = constants.substart('m4/', '%s/' % m4base, file)
 132             elif file.startswith('tests/'):
 133                 path = constants.substart('tests/', '%s/' % testsbase, file)
 134             elif file.startswith('tests=lib/'):
 135                 path = constants.substart(
 136                     'tests=lib/', '%s/' % testsbase, file)
 137             elif file.startswith('top/'):
 138                 path = constants.substart('top/', '', file)
 139             else:  # file is not a special file
 140                 path = file
 141             result += [os.path.normpath(path)]
 142         result = sorted(set(result))
 143         return(list(result))
 144
 145     def execute(self):
 146         '''GLTestDir.execute()
 147
 148         Create a scratch package with the given modules.'''
 149         localdir = self.config['localdir']
 150         auxdir = self.config['auxdir']
 151         testflags = list(self.config['testflags'])
 152         sourcebase = self.config['sourcebase']
 153         m4base = self.config['m4base']
 154         pobase = self.config['pobase']
 155         docbase = self.config['docbase']
 156         testsbase = self.config['testsbase']
 157         libname = self.config['libname']
 158         libtool = self.config['libtool']
 159         witness_c_macro = self.config['witness_c_macro']
 160         symbolic = self.config['symbolic']
 161         lsymbolic = self.config['lsymbolic']
 162         single_configure = self.config['single_configure']
 163         include_guard_prefix = self.config['include_guard_prefix']
 164         macro_prefix = self.config['macro_prefix']
 165         verbose = self.config['verbosity']
 166
 167         base_modules = [self.modulesystem.find(
 168             m) for m in self.config['modules']]
 169         if not base_modules:
 170             base_modules = self.modulesystem.list()
 171             base_modules = [self.modulesystem.find(m) for m in base_modules]
 172         # All modules together.
 173         # Except config-h, which breaks all modules which use HAVE_CONFIG_H.
 174         # Except ftruncate, mountlist, which abort the configuration on mingw.
 175         # Except lib-ignore, which leads to link errors when Sun C++ is used.
 176         base_modules = sorted(set(base_modules))
 177         base_modules = [module for module in base_modules if str(module) not in
 178                         ['config-h', 'ftruncate', 'mountlist', 'lib-ignore']]
 179
 180         # When computing transitive closures, don't consider $module to depend on
 181         # $module-tests. Need this because tests are implicitly GPL and may depend
 182         # on GPL modules - therefore we don't want a warning in this case.
 183         saved_testflags = list(self.config['testflags'])
 184         self.config.disableTestFlag(TESTS['tests'])
 185         for requested_module in base_modules:
 186             requested_licence = requested_module.getLicense()
 187             # Here we use self.moduletable.transitive_closure([module]), not just
 188             # module.getDependencies, so that we also detect weird situations like
 189             # an LGPL module which depends on a GPLed build tool module which depends
 190             # on a GPL module.
 191             if requested_licence != 'GPL':
 192                 modules = self.moduletable.transitive_closure(
 193                     [requested_module])
 194                 for module in modules:
 195                     license = module.getLicense()
 196                     errormsg = 'module %s depends on a module ' % requested_module
 197                     errormsg += 'with an incompatible license: %s\n' % module
 198                     if requested_licence == 'GPLv2+':
 199                         if license not in ['GPLv2+', 'LGPLv2+']:
 200                             sys.stderr.write(errormsg)
 201                     elif requested_licence in ['LGPL']:
 202                         if license not in ['LGPL', 'LGPLv2+']:
 203                             sys.stderr.write(errormsg)
 204                     elif requested_licence in ['LGPLv2+']:
 205                         if license not in ['LGPLv2+']:
 206                             sys.stderr.write(errormsg)
 207         self.config.setTestFlags(saved_testflags)
 208
 209         # Determine final module list.
 210         modules = self.moduletable.transitive_closure(base_modules)
 211         final_modules = list(modules)
 212
 213         # Show final module list.
 214         if verbose >= 0:
 215             bold_on = ''
 216             bold_off = ''
 217             term = os.getenv('TERM')
 218             if term == 'xterm':
 219                 bold_on = '\x1b[1m'
 220                 bold_off = '\x1b[0m'
 221             print('Module list with included dependencies (indented):')
 222             for module in final_modules:
 223                 if str(module) in self.config.getModules():
 224                     print('  %s%s%s' % (bold_on, module, bold_off))
 225                 else:  # if str(module) not in self.config.getModules()
 226                     print('    %s' % module)
 227
 228         # Generate lists of the modules.
 229         if single_configure:
 230             # Determine main module list and tests-related module list separately.
 231             main_modules, tests_modules = \
 232                 self.moduletable.transitive_closure_separately(
 233                     base_modules, final_modules)
 234             # Print main_modules and tests_modules.
 235             if verbose >= 1:
 236                 print('Main module list:')
 237                 for module in main_modules:
 238                     print('  %s' % str(module))
 239                 print('Tests-related module list:')
 240                 for module in tests_modules:
 241                     print('  %s' % str(module))
 242             # Determine whether a $testsbase/libtests.a is needed.
 243             libtests = False
 244             for module in tests_modules:
 245                 files = module.getFiles()
 246                 for file in files:
 247                     if file.startswith('lib/'):
 248                         libtests = True
 249                         break
 250             if libtests:
 251                 self.config.enableLibtests()
 252
 253         if single_configure:
 254             # Add dummy package if it is needed.
 255             main_modules = self.moduletable.add_dummy(main_modules)
 256             if 'dummy' in [str(module) for module in main_modules]:
 257                 main_modules = [m for m in main_modules if str(m) != 'dummy']
 258                 dummy = self.modulesystem.find('dummy')
 259                 main_modules = sorted(set(main_modules)) + [dummy]
 260             if libtests:  # if we need to use libtests.a
 261                 tests_modules = self.moduletable.add_dummy(tests_modules)
 262                 if 'dummy' in [str(module) for module in tests_modules]:
 263                     tests_modules = [
 264                         m for m in tests_modules if str(m) != 'dummy']
 265                     dummy = self.modulesystem.find('dummy')
 266                     tests_modules = sorted(set(tests_modules)) + [dummy]
 267         else:  # if not single_configure
 268             modules = self.moduletable.add_dummy(modules)
 269             if 'dummy' in [str(module) for module in modules]:
 270                 modules = [m for m in modules if str(m) != 'dummy']
 271                 dummy = self.modulesystem.find('dummy')
 272                 modules = sorted(set(modules)) + [dummy]
 273
 274         # Show banner notice of every module.
 275         if single_configure:
 276             for module in main_modules:
 277                 notice = module.getNotice()
 278                 if notice:
 279                     print('Notice from module %s:' % str(module))
 280                     pattern = compiler('^(.*?)$', re.S | re.M)
 281                     notice = pattern.sub('  \\1', notice)
 282                     print(notice)
 283         else:  # if not single_configure
 284             for module in modules:
 285                 notice = module.getNotice()
 286                 if notice:
 287                     print('Notice from module %s:' % str(module))
 288                     pattern = compiler('^(.*?)$', re.S | re.M)
 289                     notice = pattern.sub('  \\1', notice)
 290                     print(notice)
 291
 292         # Determine final file list.
 293         if single_configure:
 294             main_filelist, tests_filelist = \
 295                 self.moduletable.filelist_separately(
 296                     main_modules, tests_modules)
 297             filelist = sorted(set(main_filelist + tests_filelist))
 298         else:  # if not single_configure
 299             filelist = self.moduletable.filelist(modules)
 300
 301         filelist = sorted(set(filelist))
 302
 303         # Print list of files.
 304         if verbose >= 0:
 305             print('File list:')
 306             for file in filelist:
 307                 if file.startswith('tests=lib/'):
 308                     rest = file[10:]
 309                     print('  lib/%s -> tests/%s' % (rest, rest))
 310                 else:
 311                     print('  %s' % file)
 312
 313         # Add files for which the copy in gnulib is newer than the one that
 314         # "automake --add-missing --copy" would provide.
 315         filelist += ['build-aux/config.guess', 'build-aux/config.sub']
 316         filelist = sorted(set(filelist))
 317
 318         # Create directories.
 319         directories = [os.path.dirname(file)
 320                        for file in self.rewrite_files(filelist)]
 321         directories = sorted(set(directories))
 322
 323         # Copy files or make symbolic links.
 324         filetable = list()
 325         for src in filelist:
 326             dest = self.rewrite_files([src])[-1]
 327             filetable += [tuple([dest, src])]
 328         for row in filetable:
 329             src = row[1]
 330             dest = row[0]
 331             destpath = joinpath(self.testdir, dest)
 332             dirname = os.path.dirname(destpath)
 333             if not isdir(dirname):
 334                 os.makedirs(dirname)
 335             if src.startswith('tests=lib/'):
 336                 src = constants.substart('tests=lib/', 'lib/', src)
 337             lookedup, flag = self.filesystem.lookup(src)
 338             if isfile(destpath):
 339                 os.remove(destpath)
 340             if flag:
 341                 shutil.copy(lookedup, destpath)
 342             else:  # if not flag
 343                 if symbolic or (lsymbolic and lookedup == joinpath(localdir, src)):
 344                     constants.link_relative(lookedup, destpath)
 345                 else:
 346                     shutil.copy(lookedup, destpath)
 347
 348         # Create $sourcebase/Makefile.am.
 349         for_test = True
 350         directory = joinpath(self.testdir, sourcebase)
 351         if not isdir(directory):
 352             os.mkdir(directory)
 353         destfile = joinpath(directory, 'Makefile.am')
 354         if single_configure:
 355             emit, uses_subdirs = self.emiter.lib_Makefile_am(destfile, main_modules,
 356                                                              self.moduletable, self.makefiletable, '', for_test)
 357         else:  # if not single_configure
 358             emit, uses_subdirs = self.emiter.lib_Makefile_am(destfile, modules,
 359                                                              self.moduletable, self.makefiletable, '', for_test)
 360         with codecs.open(destfile, 'wb', 'UTF-8') as file:
 361             file.write(emit)
 362         any_uses_subdirs = uses_subdirs
 363
 364         # Create $m4base/Makefile.am.
 365         directory = joinpath(self.testdir, m4base)
 366         if not isdir(directory):
 367             os.mkdir(directory)
 368         destfile = joinpath(directory, 'Makefile.am')
 369         emit = string()
 370         emit += '## Process this file with automake to produce Makefile.in.\n\n'
 371         emit += 'EXTRA_DIST =\n'
 372         for file in filelist:
 373             if file.startswith('m4/'):
 374                 file = constants.substart('m4/', '', file)
 375                 emit += 'EXTRA_DIST += %s\n' % file
 376         emit = constants.nlconvert(emit)
 377         if type(emit) is bytes:
 378             emit = emit.decode(ENCS['default'])
 379         with codecs.open(destfile, 'wb', 'UTF-8') as file:
 380             file.write(emit)
 381
 382         subdirs = [sourcebase, m4base]
 383         subdirs_with_configure_ac = list()
 384
 385         testsbase_appened = False
 386         inctests = self.config.checkTestFlag(TESTS['tests'])
 387         if inctests:
 388             directory = joinpath(self.testdir, testsbase)
 389             if not isdir(directory):
 390                 os.mkdir(directory)
 391             if single_configure:
 392                 # Create $testsbase/Makefile.am.
 393                 destfile = joinpath(directory, 'Makefile.am')
 394                 print(repr(destfile))
 395                 witness_macro = '%stests_WITNESS' % macro_prefix
 396                 emit, uses_subdirs = self.emiter.tests_Makefile_am(destfile,
 397                                                                    tests_modules, self.makefiletable, witness_macro, for_test)
 398                 with codecs.open(destfile, 'wb', 'UTF-8') as file:
 399                     file.write(emit)
 400             else:  # if not single_configure
 401                 # Create $testsbase/Makefile.am.
 402                 destfile = joinpath(directory, 'Makefile.am')
 403                 libtests = False
 404                 self.config.disableLibtests()
 405                 emit, uses_subdirs = self.emiter.tests_Makefile_am(destfile,
 406                                                                    modules, self.makefiletable, '', for_test)
 407                 with codecs.open(destfile, 'wb', 'UTF-8') as file:
 408                     file.write(emit)
 409                 # Viewed from the $testsbase subdirectory, $auxdir is different.
 410                 emit = string()
 411                 saved_auxdir = self.config['auxdir']
 412                 testsbase = '%s/' % os.path.normpath(testsbase)
 413                 counter = int()
 414                 auxdir = string()
 415                 finish = (len(testsbase.split('/')) - 1)
 416                 while counter < finish:
 417                     auxdir += '../'
 418                     counter += 1
 419                 auxdir = os.path.normpath(joinpath(auxdir, saved_auxdir))
 420                 testsbase = os.path.normpath(testsbase)
 421                 self.config.setAuxDir(auxdir)
 422                 # Create $testsbase/configure.ac.
 423                 emit += '# Process this file with autoconf '
 424                 emit += 'to produce a configure script.\n'
 425                 emit += 'AC_INIT([dummy], [0])\n'
 426                 emit += 'AC_CONFIG_AUX_DIR([%s])\n' % auxdir
 427                 emit += 'AM_INIT_AUTOMAKE\n\n'
 428                 emit += 'AC_CONFIG_HEADERS([config.h])\n\n'
 429                 emit += 'AC_PROG_CC\n'
 430                 emit += 'AC_PROG_INSTALL\n'
 431                 emit += 'AC_PROG_MAKE_SET\n'
 432                 emit += 'gl_PROG_AR_RANLIB\n\n'
 433                 if uses_subdirs:
 434                     emit += 'AM_PROG_CC_C_O\n\n'
 435                 snippets = list()
 436                 for module in modules:
 437                     if str(module) in ['gnumakefile', 'maintainer-makefile']:
 438                         # These are meant to be used only in the top-level directory.
 439                         pass
 440                     # if str(module) not in ['gnumakefile', 'maintainer-makefile']
 441                     else:
 442                         snippet = module.getAutoconfSnippet_Early()
 443                         lines = [line for line in snippet.split(
 444                             '\n') if line.strip()]
 445                         snippet = '\n'.join(lines)
 446                         pattern = compiler(
 447                             'AC_REQUIRE\\(\\[([^()].*?)\\]\\)', re.S | re.M)
 448                         snippet = pattern.sub('\\1', snippet)
 449                         snippet = snippet.strip()
 450                         snippets += [snippet]
 451                 snippets = [snippet for snippet in snippets if snippet.strip()]
 452                 emit += '%s\n' % '\n'.join(snippets)
 453                 if libtool:
 454                     emit += 'LT_INIT([win32-dll])\n'
 455                     emit += 'LT_LANG([C++])\n'
 456                     emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [true])\n'
 457                     emit += 'gl_cond_libtool=true\n'
 458                 else:  # if not libtool
 459                     emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [false])\n'
 460                     emit += 'gl_cond_libtool=false\n'
 461                     emit += 'gl_libdeps=\n'
 462                     emit += 'gl_ltlibdeps=\n'
 463                 # Wrap the set of autoconf snippets into an autoconf macro that is then
 464                 # invoked. This is needed because autoconf does not support AC_REQUIRE
 465                 # at the top level:
 466                 #   error: AC_REQUIRE(gt_CSHARPCOMP): cannot be used outside of an
 467                 #     AC_DEFUN'd macro
 468                 # but we want the AC_REQUIRE to have its normal meaning (provide one
 469                 # expansion of the required macro before the current point, and only
 470                 # one expansion total).
 471                 emit += 'AC_DEFUN([gl_INIT], [\n'
 472                 replace_auxdir = True
 473                 emit += "gl_m4_base='../%s'\n" % m4base
 474                 emit += self.emiter.initmacro_start(macro_prefix)
 475                 # We don't have explicit ordering constraints between the various
 476                 # autoconf snippets. It's cleanest to put those of the library before
 477                 # those of the tests.
 478                 emit += "gl_source_base='../%s'\n" % sourcebase
 479                 emit += self.emiter.autoconfSnippets(modules,
 480                                                      self.moduletable, self.assistant, 1, False, False, False,
 481                                                      replace_auxdir)
 482                 emit += "gl_source_base='.'"
 483                 emit += self.emiter.autoconfSnippets(modules,
 484                                                      self.moduletable, self.assistant, 2, False, False, False,
 485                                                      replace_auxdir)
 486                 emit += self.emiter.initmacro_end(macro_prefix)
 487                 # _LIBDEPS and _LTLIBDEPS variables are not needed if this library is
 488                 # created using libtool, because libtool already handles the
 489                 # dependencies.
 490                 if not libtool:
 491                     libname_upper = libname.upper().replace('-', '_')
 492                     emit += '  %s_LIBDEPS="$gl_libdeps"\n' % libname_upper
 493                     emit += '  AC_SUBST([%s_LIBDEPS])\n' % libname_upper
 494                     emit += '  %s_LTLIBDEPS="$gl_ltlibdeps"\n' % libname_upper
 495                     emit += '  AC_SUBST([%s_LTLIBDEPS])\n' % libname_upper
 496                 emit += '])\n'
 497                 # FIXME use $sourcebase or $testsbase?
 498                 emit += self.emiter.initmacro_done(macro_prefix, sourcebase)
 499                 emit += '\ngl_INIT\n\n'
 500                 # Usually $testsbase/config.h will be a superset of config.h. Verify
 501                 # this by "merging" config.h into $testsbase/config.h; look out for gcc
 502                 # warnings.
 503                 emit += 'AH_TOP([#include \"../config.h\"])\n\n'
 504                 emit += 'AC_CONFIG_FILES([Makefile])\n'
 505                 emit += 'AC_OUTPUT\n'
 506                 emit = constants.nlconvert(emit)
 507                 if type(emit) is bytes:
 508                     emit = emit.decode(ENCS['default'])
 509                 path = joinpath(self.testdir, testsbase, 'configure.ac')
 510                 with codecs.open(path, 'wb', 'UTF-8') as file:
 511                     file.write(emit)
 512
 513                 # Restore changed variables.
 514                 self.config.setAuxDir(saved_auxdir)
 515                 auxdir = self.config['auxdir']
 516                 subdirs_with_configure_ac += [testsbase]
 517
 518             subdirs += [testsbase]
 519             testsbase_appened = True
 520
 521         # Create Makefile.am.
 522         emit = string()
 523         emit += '## Process this file with automake to produce Makefile.in.\n\n'
 524         emit += 'AUTOMAKE_OPTIONS = 1.9.6 foreign\n\n'
 525         emit += 'SUBDIRS = %s\n\n' % ' '.join(subdirs)
 526         emit += 'ACLOCAL_AMFLAGS = -I %s\n' % m4base
 527         emit = constants.nlconvert(emit)
 528         if type(emit) is bytes:
 529             emit = emit.decode(ENCS['default'])
 530         path = joinpath(self.testdir, 'Makefile.am')
 531         with codecs.open(path, 'wb', 'UTF-8') as file:
 532             file.write(emit)
 533
 534         # Create configure.ac
 535         emit = string()
 536         emit += '# Process this file with autoconf '
 537         emit += 'to produce a configure script.\n'
 538         emit += 'AC_INIT([dummy], [0])\n'
 539         if auxdir != '.':
 540             emit += 'AC_CONFIG_AUX_DIR([%s])\n' % auxdir
 541         emit += 'AM_INIT_AUTOMAKE\n\n'
 542         emit += 'AC_CONFIG_HEADERS([config.h])\n\n'
 543         emit += 'AC_PROG_CC\n'
 544         emit += 'AC_PROG_INSTALL\n'
 545         emit += 'AC_PROG_MAKE_SET\n\n'
 546         emit += '# For autobuild.\n'
 547         emit += 'AC_CANONICAL_BUILD\n'
 548         emit += 'AC_CANONICAL_HOST\n\n'
 549         emit += 'm4_pattern_forbid([^gl_[A-Z]])dnl the gnulib macro namespace\n'
 550         emit += 'm4_pattern_allow([^gl_ES$])dnl a valid locale name\n'
 551         emit += 'm4_pattern_allow([^gl_LIBOBJS$])dnl a variable\n'
 552         emit += 'm4_pattern_allow([^gl_LTLIBOBJS$])dnl a variable\n\n'
 553         emit += 'gl_PROG_AR_RANLIB\n\n'
 554         if any_uses_subdirs:
 555             emit += 'AM_PROG_CC_C_O\n'
 556         snippets = list()
 557         for module in final_modules:
 558             if single_configure:
 559                 solution = True
 560             else:  # if not single_configure
 561                 solution = module.isNonTests()
 562             if solution:
 563                 snippet = module.getAutoconfSnippet_Early()
 564                 lines = [line for line in snippet.split('\n') if line.strip()]
 565                 snippet = '\n'.join(lines)
 566                 pattern = compiler(
 567                     'AC_REQUIRE\\(\\[([^()].*?)\\]\\)', re.S | re.M)
 568                 snippet = pattern.sub('\\1', snippet)
 569                 snippet = snippet.strip()
 570                 snippets += [snippet]
 571         snippets = [snippet for snippet in snippets if snippet.strip()]
 572         emit += '%s\n' % '\n'.join(snippets)
 573         if libtool:
 574             emit += 'LT_INIT([win32-dll])\n'
 575             emit += 'LT_LANG([C++])\n'
 576             emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [true])\n'
 577             emit += 'gl_cond_libtool=true\n'
 578         else:  # if not libtool
 579             emit += 'AM_CONDITIONAL([GL_COND_LIBTOOL], [false])\n'
 580             emit += 'gl_cond_libtool=false\n'
 581             emit += 'gl_libdeps=\n'
 582             emit += 'gl_ltlibdeps=\n'
 583         # Wrap the set of autoconf snippets into an autoconf macro that is then
 584         # invoked. This is needed because autoconf does not support AC_REQUIRE
 585         # at the top level:
 586         #   error: AC_REQUIRE(gt_CSHARPCOMP): cannot be used outside of an
 587         #     AC_DEFUN'd macro
 588         # but we want the AC_REQUIRE to have its normal meaning (provide one
 589         # expansion of the required macro before the current point, and only one
 590         # expansion total).
 591         emit += 'AC_DEFUN([gl_INIT], [\n'
 592         if auxdir != 'build-aux':
 593             replace_auxdir = True
 594         else:  # auxdir == 'build-aux'
 595             replace_auxdir = False
 596         emit += 'gl_m4_base=\'%s\'\n' % m4base
 597         emit += self.emiter.initmacro_start(macro_prefix)
 598         emit += 'gl_source_base=\'%s\'\n' % sourcebase
 599         if single_configure:
 600             emit += self.emiter.autoconfSnippets(main_modules, self.moduletable,
 601                                                  self.assistant, 0, False, False, False, replace_auxdir)
 602         else:  # if not single_configure
 603             emit += self.emiter.autoconfSnippets(modules, self.moduletable,
 604                                                  self.assistant, 1, False, False, False, replace_auxdir)
 605         emit += self.emiter.initmacro_end(macro_prefix)
 606         if single_configure:
 607             emit += '  gltests_libdeps=\n'
 608             emit += '  gltests_ltlibdeps=\n'
 609             emit += self.emiter.initmacro_start('%stests' % macro_prefix)
 610             emit += '  gl_source_base=\'%s\'\n' % testsbase
 611             # Define a tests witness macro.
 612             emit += '  %stests_WITNESS=IN_GNULIB_TESTS\n' % macro_prefix
 613             emit += '  AC_SUBST([%stests_WITNESS])\n' % macro_prefix
 614             emit += '  gl_module_indicator_condition=$%stests_WITNESS\n' % \
 615                 macro_prefix
 616             emit += '  m4_pushdef([gl_MODULE_INDICATOR_CONDITION], '
 617             emit += '[$gl_module_indicator_condition])\n'
 618             snippets = self.emiter.autoconfSnippets(tests_modules, self.moduletable,
 619                                                     self.assistant, 1, True, False, False, replace_auxdir)
 620             emit += snippets.strip()
 621             emit += '  m4_popdef([gl_MODULE_INDICATOR_CONDITION])\n'
 622             emit += self.emiter.initmacro_end('%stests' % macro_prefix)
 623         # _LIBDEPS and _LTLIBDEPS variables are not needed if this library is
 624         # created using libtool, because libtool already handles the dependencies.
 625         if not libtool:
 626             libname_upper = libname.upper().replace('-', '_')
 627             emit += '  %s_LIBDEPS="$gl_libdeps"\n' % libname_upper
 628             emit += '  AC_SUBST([%s_LIBDEPS])\n' % libname_upper
 629             emit += '  %s_LTLIBDEPS="$gl_ltlibdeps"\n' % libname_upper
 630             emit += '  AC_SUBST([%s_LTLIBDEPS])\n' % libname_upper
 631         if single_configure and libtests:
 632             emit += '  LIBTESTS_LIBDEPS="$gltests_libdeps"\n'
 633             emit += '  AC_SUBST([LIBTESTS_LIBDEPS])\n'
 634         emit += '])\n'
 635         emit += self.emiter.initmacro_done(macro_prefix, sourcebase)
 636         if single_configure:
 637             emit += self.emiter.initmacro_done('%stests' %
 638                                                macro_prefix, testsbase)
 639         emit += '\ngl_INIT\n\n'
 640         if subdirs_with_configure_ac:
 641             if single_configure:
 642                 emit += 'AC_CONFIG_SUBDIRS([%s])\n' % \
 643                     ' '.join(subdirs_with_configure_ac[:-1])
 644             else:  # if not single_configure
 645                 emit += 'AC_CONFIG_SUBDIRS([%s])\n' % \
 646                     ' '.join(subdirs_with_configure_ac)
 647         makefiles = ['Makefile']
 648         for directory in subdirs:
 649             # For subdirs that have a configure.ac by their own, it's the subdir's
 650             # configure.ac which creates the subdir's Makefile.am, not this one.
 651             makefiles += [joinpath(directory, 'Makefile')]
 652         if not single_configure:
 653             makefiles = makefiles[:-1]
 654         emit += 'AC_CONFIG_FILES([%s])\n' % ' '.join(makefiles)
 655         emit += 'AC_OUTPUT\n'
 656         path = joinpath(self.testdir, 'configure.ac')
 657         with codecs.open(path, 'wb', 'UTF-8') as file:
 658             file.write(emit)
 659
 660         # Create autogenerated files.
 661         # Do not use "${AUTORECONF} --force --install", because it may invoke
 662         # autopoint, which brings in older versions of some of our .m4 files.
 663         os.chdir(self.testdir)
 664         # gettext
 665         if isfile(joinpath(m4base, 'gettext.m4')):
 666             args = [UTILS['autopoint'], '--force']
 667             constants.execute(args, verbose)
 668             for src in os.listdir(m4base):
 669                 src = joinpath(m4base, src)
 670                 if src.endswith('.m4~'):
 671                     dest = src[:-1]
 672                     if isfile(dest):
 673                         os.remove(dest)
 674                     shutil.move(src, dest)
 675         # libtoolize
 676         if libtool:
 677             args = [UTILS['libtoolize'], '--copy']
 678             constants.execute(args, verbose)
 679         # aclocal
 680         args = [UTILS['aclocal'], '-I', m4base]
 681         constants.execute(args, verbose)
 682         if not isdir('build-aux'):
 683             os.mkdir('build-aux')
 684         # autoconf
 685         args = [UTILS['autoconf']]
 686         constants.execute(args, verbose)
 687         # autoheader
 688         args = [UTILS['autoheader']]
 689         constants.execute(args, verbose)
 690         # automake
 691         args = [UTILS['automake'], '--add-missing', '--copy']
 692         constants.execute(args, verbose)
 693         os.chdir(DIRS['cwd'])
 694         if inctests and not single_configure:
 695             # Do not use "${AUTORECONF} --force --install", because it may invoke
 696             # autopoint, which brings in older versions of some of our .m4 files.
 697             os.chdir(joinpath(self.testdir, testsbase))
 698             # gettext
 699             if isfile(joinpath(m4base, 'gettext.m4')):
 700                 args = [UTILS['autopoint'], '--force']
 701                 constants.execute(args, verbose)
 702                 for src in os.listdir(m4base):
 703                     src = joinpath(m4base, src)
 704                     if src.endswith('.m4~'):
 705                         dest = src[:-1]
 706                         if isfile(dest):
 707                             os.remove(dest)
 708                         shutil.move(src, dest)
 709             # aclocal
 710             args = [UTILS['aclocal'], '-I', joinpath('..', m4base)]
 711             constants.execute(args, verbose)
 712             if not isdir(joinpath('../build-aux')):
 713                 os.mkdir('../build-aux')
 714             # autoconf
 715             args = [UTILS['autoconf']]
 716             constants.execute(args, verbose)
 717             # autoheader
 718             args = [UTILS['autoheader']]
 719             constants.execute(args, verbose)
 720             # automake
 721             args = [UTILS['automake'], '--add-missing', '--copy']
 722             constants.execute(args, verbose)
 723             os.chdir(DIRS['cwd'])
 724
 725         # Need to run configure and make once, to create built files that are to be
 726         # distributed (such as parse-datetime.c).
 727         path = joinpath(self.testdir, sourcebase, 'Makefile.am')
 728         with codecs.open(path, 'rb', 'UTF-8') as file:
 729             snippet = file.read()
 730         snippet = constants.remove_backslash_newline(snippet)
 731         cleaned_files = list()
 732         tests_cleaned_files = list()
 733         built_sources = list()
 734         tests_built_sources = list()
 735         distributed_built_sources = list()
 736         tests_distributed_built_sources = list()
 737
 738         # Extract the value of "CLEANFILES += ..." and "MOSTLYCLEANFILES += ...".
 739         regex_find = list()
 740         pattern = compiler('^CLEANFILES[\t ]*\\+=(.*?)$', re.S | re.M)
 741         regex_find += pattern.findall(snippet)
 742         pattern = compiler('^MOSTLYCLEANFILES[\t ]*\\+=(.*?)$', re.S | re.M)
 743         regex_find += pattern.findall(snippet)
 744         regex_find = [line.strip() for line in regex_find if line.strip()]
 745         for part in regex_find:
 746             cleaned_files += \
 747                 [line.strip() for line in part.split(' ') if line.strip()]
 748
 749         # Extract the value of "BUILT_SOURCES += ...". Remove variable references
 750         # such $(FOO_H) because they don't refer to distributed files.
 751         regex_find = list()
 752         pattern = compiler('^BUILT_SOURCES[\t ]*\\+=(.*?)$', re.S | re.M)
 753         regex_find += pattern.findall(snippet)
 754         regex_find = [line.strip() for line in regex_find if line.strip()]
 755         for part in regex_find:
 756             built_sources += \
 757                 [line.strip() for line in part.split(' ') if line.strip()]
 758         built_sources = [line for line in built_sources
 759                          if not bool(compiler('[$]\\([A-Za-z0-9_]*\\)$').findall(line))]
 760         distributed_built_sources = [file for file in built_sources
 761                                      if file not in cleaned_files]
 762
 763         if inctests:
 764             # Likewise for built files in the $testsbase directory.
 765             path = joinpath(self.testdir, testsbase, 'Makefile.am')
 766             with codecs.open(path, 'rb', 'UTF-8') as file:
 767                 snippet = file.read()
 768             snippet = constants.remove_backslash_newline(snippet)
 769
 770             # Extract the value of "CLEANFILES += ..." and "MOSTLYCLEANFILES += ...".
 771             regex_find = list()
 772             pattern = compiler('^CLEANFILES[\t ]*\\+=(.*?)$', re.S | re.M)
 773             regex_find += pattern.findall(snippet)
 774             pattern = compiler(
 775                 '^MOSTLYCLEANFILES[\t ]*\\+=(.*?)$', re.S | re.M)
 776             regex_find += pattern.findall(snippet)
 777             regex_find = [line.strip() for line in regex_find if line.strip()]
 778             for part in regex_find:
 779                 tests_cleaned_files += \
 780                     [line.strip() for line in part.split(' ') if line.strip()]
 781
 782             # Extract the value of "BUILT_SOURCES += ...". Remove variable references
 783             # such $(FOO_H) because they don't refer to distributed files.
 784             regex_find = list()
 785             tests_built_sources = list()
 786             pattern = compiler('^BUILT_SOURCES[\t ]*\\+=(.*?)$', re.S | re.M)
 787             regex_find += pattern.findall(snippet)
 788             regex_find = [line.strip() for line in regex_find if line.strip()]
 789             for part in regex_find:
 790                 tests_built_sources += \
 791                     [line.strip() for line in part.split(' ') if line.strip()]
 792             tests_built_sources = [line for line in tests_built_sources
 793                                    if not bool(compiler('[$]\\([A-Za-z0-9_]*\\)$').findall(line))]
 794             tests_distributed_built_sources = [file for file in tests_built_sources
 795                                                if file not in cleaned_files]
 796
 797         if distributed_built_sources or tests_distributed_built_sources:
 798             os.chdir(self.testdir)
 799             sp.call('./configure')
 800             if distributed_built_sources:
 801                 os.chdir(sourcebase)
 802                 with codecs.open('Makefile', 'ab', 'UTF-8') as file:
 803                     file.write('built_sources: $(BUILT_SOURCES)\n')
 804                 args = [UTILS['make'],
 805                         'AUTOCONF=%s' % UTILS['autoconf'],
 806                         'AUTOHEADER=%s' % UTILS['autoheader'],
 807                         'ACLOCAL=%s' % UTILS['aclocal'],
 808                         'AUTOMAKE=%s' % UTILS['automake'],
 809                         'AUTORECONF=%s' % UTILS['autoreconf'],
 810                         'built_sources']
 811                 sp.call(args)
 812                 os.chdir('..')
 813             if tests_distributed_built_sources:
 814                 os.chdir(testsbase)
 815                 with codecs.open('Makefile', 'ab', 'UTF-8') as file:
 816                     file.write('built_sources: $(BUILT_SOURCES)\n')
 817                 args = [UTILS['make'],
 818                         'AUTOCONF=%s' % UTILS['autoconf'],
 819                         'AUTOHEADER=%s' % UTILS['autoheader'],
 820                         'ACLOCAL=%s' % UTILS['aclocal'],
 821                         'AUTOMAKE=%s' % UTILS['automake'],
 822                         'AUTORECONF=%s' % UTILS['autoreconf'],
 823                         'built_sources']
 824                 sp.call(args)
 825                 os.chdir('..')
 826             args = [UTILS['make'],
 827                     'AUTOCONF=%s' % UTILS['autoconf'],
 828                     'AUTOHEADER=%s' % UTILS['autoheader'],
 829                     'ACLOCAL=%s' % UTILS['aclocal'],
 830                     'AUTOMAKE=%s' % UTILS['automake'],
 831                     'AUTORECONF=%s' % UTILS['autoreconf'],
 832                     'AUTOPOINT=%s' % UTILS['autopoint'],
 833                     'LIBTOOLIZE=%s' % UTILS['libtoolize'],
 834                     'distclean']
 835             sp.call(args)
 836         sp.call(['rm', '-rf', self.config['tempdir']], shell=False)
 837
 838
 839 #===============================================================================
 840 # Define GLMegaTestDir class
 841 #===============================================================================
 842 class GLMegaTestDir(object):
 843     '''GLMegaTestDir class is used to create a mega scratch package with the
 844     given modules one by one and all together.'''
 845
 846     def __init__(self, config, megatestdir):
 847         '''GLMegaTestDir.__init__(config, megatestdir) -> GLMegaTestDir
 848
 849         Create new GLTestDir instance.'''
 850         if type(config) is not GLConfig:
 851             raise(TypeError('config must be a GLConfig, not %s' %
 852                             type(config).__name__))
 853         if type(megatestdir) is bytes or type(megatestdir) is string:
 854             if type(megatestdir) is bytes:
 855                 megatestdir = megatestdir.decode(ENCS['default'])
 856         self.config = config
 857         self.megatestdir = os.path.normpath(megatestdir)
 858         if not os.path.exists(self.megatestdir):
 859             try:  # Try to create directory
 860                 os.mkdir(self.megatestdir)
 861             except Exception as error:
 862                 raise(GLError(19, self.megatestdir))
 863         self.emiter = GLEmiter(self.config)
 864         self.filesystem = GLFileSystem(self.config)
 865         self.modulesystem = GLModuleSystem(self.config)
 866         self.moduletable = GLModuleTable(self.config)
 867         self.assistant = GLFileAssistant(self.config)
 868         self.makefiletable = GLMakefileTable(self.config)
 869
 870     def execute(self):
 871         '''GLMegaTestDir.execute()
 872
 873         Create a mega scratch package with the given modules one by one and all
 874         together.'''
 875         megasubdirs = list()
 876         modules = [self.modulesystem.find(m) for m in self.config['modules']]
 877         if not modules:
 878             modules = self.modulesystem.list()
 879             modules = [self.modulesystem.find(m) for m in modules]
 880         modules = sorted(set(modules))
 881
 882         # First, all modules one by one.
 883         for module in modules:
 884             self.config.setModules([str(module)])
 885             #GLTestDir(self.config, self.megatestdir).execute()
 886             megasubdirs += [str(module)]
 887
 888         # Then, all modules all together.
 889         # Except config-h, which breaks all modules which use HAVE_CONFIG_H.
 890         modules = [module for module in modules if str(module) != 'config-h']
 891         self.config.setModules([str(module) for module in modules])
 892         #GLTestDir(self.config, self.megatestdir).execute()
 893         megasubdirs += ['ALL']
 894
 895         # Create autobuild.
 896         emit = string()
 897         repdict = dict()
 898         repdict['Jan'] = repdict['January'] = '01'
 899         repdict['Feb'] = repdict['February'] = '02'
 900         repdict['Mar'] = repdict['March'] = '03'
 901         repdict['Apr'] = repdict['April'] = '04'
 902         repdict['May'] = repdict['May'] = '05'
 903         repdict['Jun'] = repdict['June'] = '06'
 904         repdict['Jul'] = repdict['July'] = '07'
 905         repdict['Aug'] = repdict['August'] = '08'
 906         repdict['Sep'] = repdict['September'] = '09'
 907         repdict['Oct'] = repdict['October'] = '10'
 908         repdict['Nov'] = repdict['November'] = '11'
 909         repdict['Dec'] = repdict['December'] = '12'
 910         vc_witness = joinpath(DIRS['root'], '.git', 'refs', 'heads', 'master')
 911         mdate_sh = joinpath(DIRS['root'], 'build-aux', 'mdate-sh')
 912         args = ['sh', mdate_sh, vc_witness]
 913         cvsdate = sp.check_output(args).decode("UTF-8").strip()
 914         for key in repdict:
 915             if len(key) > 3:
 916                 cvsdate = cvsdate.replace(key, repdict[key])
 917         for key in repdict:
 918             cvsdate = cvsdate.replace(key, repdict[key])
 919         cvsdate = ''.join(
 920             [date for date in cvsdate.split(' ') if date.strip()])
 921         cvsdate = '%s%s%s' % (cvsdate[4:], cvsdate[2:4], cvsdate[:2])
 922         emit += '#!/bin/sh\n'
 923         emit += 'CVSDATE=%s\n' % cvsdate
 924         emit += ': ${MAKE=make}\n'
 925         emit += 'test -d logs || mkdir logs\n'
 926         emit += 'for module in %s; do\n' % ' '.join(megasubdirs)
 927         emit += '  echo "Working on module $module..."\n'
 928         emit += '  safemodule=`echo $module | sed -e \'s|/|-|g\'`\n'
 929         emit += '  (echo "To: gnulib@autobuild.josefsson.org"\\\n'
 930         emit += '   echo\n'
 931         emit += '   set -x\n'
 932         emit += '   : autobuild project... $module\n'
 933         emit += '   : autobuild revision... cvs-$CVSDATE-000000\n'
 934         emit += '   : autobuild timestamp... `date "+%Y%m%d-%H%M%S"`\n'
 935         emit += '   : autobuild hostname... `hostname`\n'
 936         emit += '   cd $module && ./configure $CONFIGURE_OPTIONS && $MAKE'
 937         emit += ' && $MAKE check && $MAKE distclean\n'
 938         emit += '   echo rc=$?\n'
 939         emit += '  ) 2>&1 | { if test -n "$AUTOBUILD_SUBST"; then '
 940         emit += 'sed -e "$AUTOBUILD_SUBST"; else cat; fi; } > logs/$safemodule\n'
 941         emit += 'done\n'
 942         emit = constants.nlconvert(emit)
 943         if type(emit) is bytes:
 944             emit = emit.decode(ENCS['default'])
 945         path = joinpath(self.megatestdir, 'do-autobuild')
 946         with codecs.open(path, 'wb', 'UTF-8') as file:
 947             file.write(emit)
 948
 949         # Create Makefile.am.
 950         emit = string()
 951         emit += '## Process this file with automake to produce Makefile.in.\n\n'
 952         emit += 'AUTOMAKE_OPTIONS = 1.9.6 foreign\n\n'
 953         emit += 'SUBDIRS = %s\n\n' % ' '.join(megasubdirs)
 954         emit += 'EXTRA_DIST = do-autobuild\n'
 955         emit = constants.nlconvert(emit)
 956         if type(emit) is bytes:
 957             emit = emit.decode(ENCS['default'])
 958         path = joinpath(self.megatestdir, 'Makefile.am')
 959         with codecs.open(path, 'wb', 'UTF-8') as file:
 960             file.write(emit)
 961
 962         emit = string()
 963         emit += '# Process this file with autoconf '
 964         emit += 'to produce a configure script.\n'
 965         emit += 'AC_INIT([dummy], [0])\n\n'
 966         if auxdir != '.':
 967             emit += 'AC_CONFIG_AUX_DIR([%s])\n' % auxdir
 968         emit += 'AM_INIT_AUTOMAKE\n\n'
 969         emit += 'AC_PROG_MAKE_SET\n\n'
 970         emit += 'AC_CONFIG_SUBDIRS([%s])\n' % ' '.megasubdirs
 971         emit += 'AC_CONFIG_FILES([Makefile])\n'
 972         emit += 'AC_OUTPUT\n'
 973         emit = constants.nlconvert(emit)
 974         if type(emit) is bytes:
 975             emit = emit.decode(ENCS['default'])
 976         path = joinpath(self.megatestdir, 'Makefile.am')
 977         with codecs.open(path, 'wb', 'UTF-8') as file:
 978             file.write(emit)
 979
 980         # Create autogenerated files.
 981         os.chdir(DIRS['cwd'])
 982         args = [UTILS['aclocal']]
 983         constants.execute(args, verbose)
 984         try:  # Try to make a directory
 985             if not isdir('build-aux'):
 986                 os, mkdir('build-aux')
 987         except Exception as error:
 988             pass
 989         args = [UTILS['autoconf']]
 990         constants.execute(args, verbose)
 991         args = [UTILS['automake'], '--add-missing', '--copy']
 992         constants.execute(args, verbose)
 993         sp.call(['rm', '-rf', self.config['tempdir']], shell=False)