pygnulib/GLImport.py

   1 #!/usr/bin/python
   2 # encoding: UTF-8
   3
   4 #===============================================================================
   5 # Define global imports
   6 #===============================================================================
   7 import os
   8 import re
   9 import sys
  10 import locale
  11 import codecs
  12 import shutil
  13 import filecmp
  14 import subprocess as sp
  15 from . import constants
  16 from .GLError import GLError
  17 from .GLConfig import GLConfig
  18 from .GLModuleSystem import GLModule
  19 from .GLModuleSystem import GLModuleTable
  20 from .GLModuleSystem import GLModuleSystem
  21 from .GLFileSystem import GLFileSystem
  22 from .GLFileSystem import GLFileAssistant
  23 from .GLMakefileTable import GLMakefileTable
  24 from .GLEmiter import GLEmiter
  25
  26
  27 #===============================================================================
  28 # Define module information
  29 #===============================================================================
  30 __author__ = constants.__author__
  31 __license__ = constants.__license__
  32 __copyright__ = constants.__copyright__
  33
  34
  35 #===============================================================================
  36 # Define global constants
  37 #===============================================================================
  38 PYTHON3 = constants.PYTHON3
  39 NoneType = type(None)
  40 APP = constants.APP
  41 DIRS = constants.DIRS
  42 ENCS = constants.ENCS
  43 UTILS = constants.UTILS
  44 MODES = constants.MODES
  45 TESTS = constants.TESTS
  46 compiler = constants.compiler
  47 joinpath = constants.joinpath
  48 cleaner = constants.cleaner
  49 relpath = constants.relativize
  50 string = constants.string
  51 isabs = os.path.isabs
  52 isdir = os.path.isdir
  53 isfile = os.path.isfile
  54 normpath = os.path.normpath
  55
  56
  57 #===============================================================================
  58 # Define GLImport class
  59 #===============================================================================
  60 class GLImport(object):
  61     '''GLImport class is used to provide methods for --import, --add-import,
  62     --remove-import and --update actions. This is a high-level class, so
  63     developers may  have to use lower-level classes to create their own
  64     scripts. However, if user needs just to use power of gnulib-tool, this class
  65     is a very good choice.'''
  66
  67     def __init__(self, config, mode):
  68         '''Create GLImport instance.
  69         The first variable, mode, must be one of the values of the MODES dict
  70         object, which is accessible from constants module. The second one, config,
  71         must be a GLConfig object.'''
  72         if type(config) is not GLConfig:
  73             raise(TypeError('config must have GLConfig type, not %s' %
  74                             repr(config)))
  75         if type(mode) is int and \
  76                 MODES['import'] <= mode <= MODES['update']:
  77             self.mode = mode
  78         else:  # if mode is not int or is not 0-3
  79             raise(TypeError('mode must be 0 <= mode <= 3, not %s' %
  80                             repr(mode)))
  81
  82         # Initialize some values.
  83         self.cache = GLConfig()
  84         self.config = config.copy()
  85         os.rmdir(self.cache['tempdir'])
  86
  87         # Get cached auxdir and libtool from configure.ac/in.
  88         self.cache.setAuxDir('.')
  89         path = joinpath(self.config['destdir'], 'configure.ac')
  90         if not isfile(path):
  91             path = joinpath(self.config['destdir'], 'configure.in')
  92             if not isfile(path):
  93                 raise(GLError(3, path))
  94         self.config.setAutoconfFile(path)
  95         with codecs.open(path, 'rb', 'UTF-8') as file:
  96             data = file.read()
  97         pattern = compiler(r'^AC_CONFIG_AUX_DIR\((.*?)\)$', re.S | re.M)
  98         match = pattern.findall(data)
  99         if match:
 100             result = cleaner(match)[0]
 101             self.cache.setAuxDir(joinpath(result, self.config['destdir']))
 102         pattern = compiler(r'A[CM]_PROG_LIBTOOL', re.S | re.M)
 103         guessed_libtool = bool(pattern.findall(data))
 104         if self.config['auxdir'] == None:
 105             self.config.setAuxDir(self.cache['auxdir'])
 106
 107         # Guess autoconf version.
 108         pattern = compiler('.*AC_PREREQ\((.*?)\)', re.S | re.M)
 109         versions = cleaner(pattern.findall(data))
 110         if versions:
 111             version = sorted(set([float(version) for version in versions]))[-1]
 112             self.config.setAutoconfVersion(version)
 113             if version < 2.59:
 114                 raise(GLError(4, version))
 115
 116         # Get other cached variables.
 117         path = joinpath(self.config['m4base'], 'gnulib-cache.m4')
 118         if isfile(joinpath(self.config['m4base'], 'gnulib-cache.m4')):
 119             with codecs.open(path, 'rb', 'UTF-8') as file:
 120                 data = file.read()
 121
 122             # Create regex object and keys.
 123             pattern = compiler('^(gl_.*?)\\((.*?)\\)$', re.S | re.M)
 124             keys = \
 125                 [
 126                     'gl_LOCAL_DIR', 'gl_MODULES', 'gl_AVOID', 'gl_SOURCE_BASE',
 127                     'gl_M4_BASE', 'gl_PO_BASE', 'gl_DOC_BASE', 'gl_TESTS_BASE',
 128                     'gl_MAKEFILE_NAME', 'gl_MACRO_PREFIX', 'gl_PO_DOMAIN',
 129                     'gl_WITNESS_C_MACRO', 'gl_VC_FILES', 'gl_LIB',
 130                 ]
 131
 132             # Find bool values.
 133             if 'gl_LGPL(' in data:
 134                 keys.append('gl_LGPL')
 135                 self.cache.setLGPL(True)
 136             if 'gl_LIBTOOL' in data:
 137                 self.cache.enableLibtool()
 138                 data = data.replace('gl_LIBTOOL', '')
 139             if 'gl_CONDITIONAL_DEPENDENCIES' in data:
 140                 self.cache.enableCondDeps()
 141                 data = data.replace('gl_CONDITIONAL_DEPENDENCIES', '')
 142             if 'gl_VC_FILES' in data:
 143                 self.cache.enableVCFiles()
 144                 data = data.replace('gl_VC_FILES', '')
 145             if 'gl_WITH_TESTS' in data:
 146                 self.cache.enableTestFlag(TESTS['tests'])
 147                 data = data.replace('gl_WITH_TESTS', '')
 148             if 'gl_WITH_OBSOLETE' in data:
 149                 self.cache.enableTestFlag(TESTS['obsolete'])
 150                 data = data.replace('gl_WITH_OBSOLETE', '')
 151             if 'gl_WITH_CXX_TESTS' in data:
 152                 self.cache.enableTestFlag(TESTS['c++-test'])
 153                 data = data.replace('gl_WITH_CXX_TESTS', '')
 154             if 'gl_WITH_LONGRUNNING_TESTS' in data:
 155                 self.cache.enableTestFlag(TESTS['longrunning-test'])
 156                 data = data.replace('gl_WITH_LONGRUNNING_TESTS', '')
 157             if 'gl_WITH_PRIVILEGED_TESTS' in data:
 158                 self.cache.enableTestFlag(TESTS['privileged-test'])
 159                 data = data.replace('gl_WITH_PRIVILEGED_TESTS', '')
 160             if 'gl_WITH_UNPORTABLE_TESTS' in data:
 161                 self.cache.enableTestFlag(TESTS['unportable-test'])
 162                 data = data.replace('gl_WITH_UNPORTABLE_TESTS', '')
 163             if 'gl_WITH_ALL_TESTS' in data:
 164                 self.cache.enableTestFlag(TESTS['all-test'])
 165                 data = data.replace('gl_WITH_ALL_TESTS', '')
 166             # Find string values
 167             result = dict(pattern.findall(data))
 168             values = cleaner([result.get(key, '') for key in keys])
 169             tempdict = dict(zip(keys, values))
 170             if 'gl_LGPL' in tempdict:
 171                 lgpl = cleaner(tempdict['gl_LGPL'])
 172                 if lgpl.isdecimal():
 173                     self.cache.setLGPL(int(self.cache['lgpl']))
 174             else:  # if 'gl_LGPL' not in tempdict
 175                 self.cache.setLGPL(False)
 176             if tempdict['gl_LIB']:
 177                 self.cache.setLibName(cleaner(tempdict['gl_LIB']))
 178             if tempdict['gl_LOCAL_DIR']:
 179                 self.cache.setLocalDir(cleaner(tempdict['gl_LOCAL_DIR']))
 180             if tempdict['gl_MODULES']:
 181                 self.cache.setModules(cleaner(tempdict['gl_MODULES'].split()))
 182             if tempdict['gl_AVOID']:
 183                 self.cache.setAvoids(cleaner(tempdict['gl_AVOID'].split()))
 184             if tempdict['gl_SOURCE_BASE']:
 185                 self.cache.setSourceBase(cleaner(tempdict['gl_SOURCE_BASE']))
 186             if tempdict['gl_M4_BASE']:
 187                 self.cache.setM4Base(cleaner(tempdict['gl_M4_BASE']))
 188             if tempdict['gl_PO_BASE']:
 189                 self.cache.setPoBase(cleaner(tempdict['gl_PO_BASE']))
 190             if tempdict['gl_DOC_BASE']:
 191                 self.cache.setDocBase(cleaner(tempdict['gl_DOC_BASE']))
 192             if tempdict['gl_TESTS_BASE']:
 193                 self.cache.setTestsBase(cleaner(tempdict['gl_TESTS_BASE']))
 194             if tempdict['gl_MAKEFILE_NAME']:
 195                 self.cache.setMakefile(cleaner(tempdict['gl_MAKEFILE_NAME']))
 196             if tempdict['gl_MACRO_PREFIX']:
 197                 self.cache.setMacroPrefix(cleaner(tempdict['gl_MACRO_PREFIX']))
 198             if tempdict['gl_PO_DOMAIN']:
 199                 self.cache.setPoDomain(cleaner(tempdict['gl_PO_DOMAIN']))
 200             if tempdict['gl_WITNESS_C_MACRO']:
 201                 self.cache.setWitnessCMacro(
 202                     cleaner(tempdict['gl_WITNESS_C_MACRO']))
 203
 204             # Get cached filelist from gnulib-comp.m4.
 205             destdir, m4base = self.config.getDestDir(), self.config.getM4Base()
 206             path = joinpath(destdir, m4base, 'gnulib-comp.m4')
 207             if isfile(path):
 208                 with codecs.open(path, 'rb', 'UTF-8') as file:
 209                     data = file.read()
 210                 regex = 'AC_DEFUN\\(\\[%s_FILE_LIST\\], \\[(.*?)\\]\\)' % \
 211                     self.cache['macro_prefix']
 212                 pattern = compiler(regex, re.S | re.M)
 213                 self.cache.setFiles(pattern.findall(data)[-1].strip().split())
 214
 215         # The self.config['localdir'] defaults to the cached one. Recall that the
 216         # cached one is relative to $destdir, whereas the one we use is relative
 217         # to . or absolute.
 218         if not self.config['localdir']:
 219             if self.cache['localdir']:
 220                 if isabs(self.config['destdir']):
 221                     localdir = joinpath(
 222                         self.config['destdir'], self.cache['localdir'])
 223                 else:  # if not isabs(self.config['destdir'])
 224                     if isabs(self.cache['localdir']):
 225                         localdir = joinpath(
 226                             self.config['destdir'], self.cache['localdir'])
 227                     else:  # if not isabs(self.cache['localdir'])
 228                         # NOTE: I NEED TO IMPLEMENT RELATIVE_CONCAT
 229                         localdir = os.path.relpath(joinpath(self.config['destdir'],
 230                                                             self.cache['localdir']))
 231                 self.config.setLocalDir(localdir)
 232
 233         if self.mode != MODES['import']:
 234             if self.cache['m4base'] and \
 235                     (self.config['m4base'] != self.cache['m4base']):
 236                 raise(GLError(5, m4base))
 237
 238             # Perform actions with modules. In --add-import, append each given module
 239             # to the list of cached modules; in --remove-import, remove each given
 240             # module from the list of cached modules; in --update, simply set
 241             # self.config['modules'] to its cached version.
 242             new, old = self.config.getModules(), self.cache.getModules()
 243             if self.mode == MODES['add-import']:
 244                 modules = sorted(set(new + old))
 245             elif self.mode == MODES['remove-import']:
 246                 modules = [module for module in old if module in new]
 247             elif self.mode == MODES['update']:
 248                 modules = self.cache.getModules()
 249
 250             # If user tries to apply conddeps and testflag['tests'] together.
 251             if self.config['tests'] and self.config['conddeps']:
 252                 raise(GLError(10, None))
 253
 254             # Update configuration dictionary.
 255             self.config.update(self.cache)
 256             for key in config.keys():
 257                 value = config[key]
 258                 if not config.isdefault(key, value):
 259                     self.config.update_key(config, key)
 260             self.config.setModules(modules)
 261
 262         # Check if conddeps is enabled together with inctests.
 263         inctests = self.config.checkTestFlag(TESTS['tests'])
 264         if self.config['conddeps'] and inctests:
 265             raise(GLError(10, None))
 266
 267         # Define GLImport attributes.
 268         self.emiter = GLEmiter(self.config)
 269         self.filesystem = GLFileSystem(self.config)
 270         self.modulesystem = GLModuleSystem(self.config)
 271         self.moduletable = GLModuleTable(self.config, list())
 272         self.makefiletable = GLMakefileTable(self.config)
 273
 274     def __repr__(self):
 275         '''x.__repr__ <==> repr(x)'''
 276         result = '<pygnulib.GLImport %s>' % hex(id(self))
 277         return(result)
 278
 279     def rewrite_old_files(self, files):
 280         '''GLImport.rewrite_old_files(files) -> list
 281
 282         Replace auxdir, docbase, sourcebase, m4base and testsbase from default
 283         to their version from cached config.'''
 284         if type(files) is not list:
 285             raise(TypeError(
 286                 'files argument must has list type, not %s' % type(files).__name__))
 287         files = \
 288             [  # Begin to convert bytes to string
 289                 file.decode(ENCS['default']) \
 290                 if type(file) is bytes else file \
 291                 for file in files
 292             ]  # Finish to convert bytes to string
 293         for file in files:
 294             if type(file) is not string:
 295                 raise(TypeError('each file must be a string instance'))
 296         files = sorted(set(files))
 297         files = ['%s%s' % (file, os.path.sep) for file in files]
 298         auxdir = self.cache['auxdir']
 299         docbase = self.cache['docbase']
 300         sourcebase = self.cache['sourcebase']
 301         m4base = self.cache['m4base']
 302         testsbase = self.cache['testsbase']
 303         result = list()
 304         for file in files:
 305             if file.startswith('build-aux/'):
 306                 path = constants.substart('build-aux/', '%s/' % auxdir, file)
 307             elif file.startswith('doc/'):
 308                 path = constants.substart('doc/', '%s/' % docbase, file)
 309             elif file.startswith('lib/'):
 310                 path = constants.substart('lib/', '%s/' % sourcebase, file)
 311             elif file.startswith('m4/'):
 312                 path = constants.substart('m4/', '%s/' % m4base, file)
 313             elif file.startswith('tests/'):
 314                 path = constants.substart('tests/', '%s/' % testsbase, file)
 315             elif file.startswith('tests=lib/'):
 316                 path = constants.substart(
 317                     'tests=lib/', '%s/' % testsbase, file)
 318             elif file.startswith('top/'):
 319                 path = constants.substart('top/', '', file)
 320             else:  # file is not a special file
 321                 path = file
 322             result += [os.path.normpath(path)]
 323         result = sorted(set(result))
 324         return(list(result))
 325
 326     def rewrite_new_files(self, files):
 327         '''GLImport.rewrite_new_files(files)
 328
 329         Replace auxdir, docbase, sourcebase, m4base and testsbase from default
 330         to their version from config.'''
 331         if type(files) is not list:
 332             raise(TypeError(
 333                 'files argument must has list type, not %s' % type(files).__name__))
 334         files = \
 335             [  # Begin to convert bytes to string
 336                 file.decode(ENCS['default']) \
 337                 if type(file) is bytes else file \
 338                 for file in files
 339             ]  # Finish to convert bytes to string
 340         for file in files:
 341             if type(file) is not string:
 342                 raise(TypeError('each file must be a string instance'))
 343         files = sorted(set(files))
 344         auxdir = self.config['auxdir']
 345         docbase = self.config['docbase']
 346         sourcebase = self.config['sourcebase']
 347         m4base = self.config['m4base']
 348         testsbase = self.config['testsbase']
 349         result = list()
 350         for file in files:
 351             if file.startswith('build-aux/'):
 352                 path = constants.substart('build-aux/', '%s/' % auxdir, file)
 353             elif file.startswith('doc/'):
 354                 path = constants.substart('doc/', '%s/' % docbase, file)
 355             elif file.startswith('lib/'):
 356                 path = constants.substart('lib/', '%s/' % sourcebase, file)
 357             elif file.startswith('m4/'):
 358                 path = constants.substart('m4/', '%s/' % m4base, file)
 359             elif file.startswith('tests/'):
 360                 path = constants.substart('tests/', '%s/' % testsbase, file)
 361             elif file.startswith('tests=lib/'):
 362                 path = constants.substart(
 363                     'tests=lib/', '%s/' % testsbase, file)
 364             elif file.startswith('top/'):
 365                 path = constants.substart('top/', '', file)
 366             else:  # file is not a special file
 367                 path = file
 368             result += [os.path.normpath(path)]
 369         result = sorted(set(result))
 370         return(list(result))
 371
 372     def actioncmd(self):
 373         '''Return command-line invocation comment.'''
 374         modules = self.config.getModules()
 375         avoids = self.config.getAvoids()
 376         destdir = self.config.getDestDir()
 377         localdir = self.config.getLocalDir()
 378         auxdir = self.config.getAuxDir()
 379         sourcebase = self.config.getSourceBase()
 380         m4base = self.config.getM4Base()
 381         docbase = self.config.getDocBase()
 382         pobase = self.config.getPoBase()
 383         testsbase = self.config.getTestsBase()
 384         testflags = self.config.getTestFlags()
 385         conddeps = self.config.checkCondDeps()
 386         libname = self.config.getLibName()
 387         lgpl = self.config.getLGPL()
 388         makefile = self.config.getMakefile()
 389         libtool = self.config.checkLibtool()
 390         macro_prefix = self.config.getMacroPrefix()
 391         witness_c_macro = self.config.getWitnessCMacro()
 392         podomain = self.config.getPoDomain()
 393         vc_files = self.config.checkVCFiles()
 394         verbose = self.config.getVerbosity()
 395
 396         # Create command-line invocation comment.
 397         actioncmd = 'gnulib-tool --import'
 398         actioncmd += ' --dir=%s' % destdir
 399         if localdir:
 400             actioncmd += ' --local-dir=%s' % localdir
 401         actioncmd += ' --lib=%s' % libname
 402         actioncmd += ' --source-base=%s' % sourcebase
 403         actioncmd += ' --m4-base=%s' % m4base
 404         if pobase:
 405             actioncmd += ' --po-base=%s' % pobase
 406         actioncmd += ' --doc-base=%s' % docbase
 407         actioncmd += ' --tests-base=%s' % testsbase
 408         actioncmd += ' --aux-dir=%s' % auxdir
 409         if self.config.checkTestFlag(TESTS['tests']):
 410             actioncmd += ' --with-tests'
 411         if self.config.checkTestFlag(TESTS['obsolete']):
 412             actioncmd += ' --with-obsolete'
 413         if self.config.checkTestFlag(TESTS['c++-test']):
 414             actioncmd += ' --with-c++-tests'
 415         if self.config.checkTestFlag(TESTS['longrunning-test']):
 416             actioncmd += ' --with-longrunning-tests'
 417         if self.config.checkTestFlag(TESTS['privileged-test']):
 418             actioncmd += ' --with-privileged-test'
 419         if self.config.checkTestFlag(TESTS['unportable-test']):
 420             actioncmd += ' --with-unportable-tests'
 421         if self.config.checkTestFlag(TESTS['all-test']):
 422             actioncmd += ' --with-all-tests'
 423         for module in avoids:
 424             actioncmd += ' --avoid=%s' % module
 425         if lgpl:
 426             if lgpl == True:
 427                 actioncmd += ' --lgpl'
 428             else:  # if lgpl != True
 429                 actioncmd += ' --lgpl=%s' % lgpl
 430         if makefile:
 431             actioncmd += ' --makefile-name=%s' % makefile
 432         if conddeps:
 433             actioncmd += ' --conditional-dependencies'
 434         else:  # if not conddeps
 435             actioncmd += ' --no-conditional-dependencies'
 436         if libtool:
 437             actioncmd += ' --libtool'
 438         else:  # if not libtool
 439             actioncmd += ' --no-libtool'
 440         actioncmd += ' --macro-prefix=%s' % macro_prefix
 441         if podomain:
 442             actioncmd = ' --podomain=%s' % podomain
 443         if witness_c_macro:
 444             actioncmd += ' --witness_c_macro=%s' % witness_c_macro
 445         if vc_files == True:
 446             actioncmd += ' --vc-files'
 447         elif vc_files == False:
 448             actioncmd += ' --no-vc-files'
 449         actioncmd += ' '  # Add a space
 450         actioncmd += ' '.join(modules)
 451         return(actioncmd)
 452
 453     def gnulib_cache(self):
 454         '''GLImport.gnulib_cache() -> string
 455
 456         Emit the contents of generated $m4base/gnulib-cache.m4 file.
 457         GLConfig: destdir, localdir, tests, sourcebase, m4base, pobase, docbase,
 458         testsbase, conddeps, libtool, macro_prefix, podomain, vc_files.'''
 459         emit = string()
 460         moduletable = self.moduletable
 461         actioncmd = self.actioncmd()
 462         destdir = self.config['destdir']
 463         localdir = self.config['localdir']
 464         testflags = list(self.config['testflags'])
 465         sourcebase = self.config['sourcebase']
 466         m4base = self.config['m4base']
 467         pobase = self.config['pobase']
 468         docbase = self.config['docbase']
 469         testsbase = self.config['testsbase']
 470         lgpl = self.config['lgpl']
 471         libname = self.config['libname']
 472         makefile = self.config['makefile']
 473         conddeps = self.config['conddeps']
 474         libtool = self.config['libtool']
 475         macro_prefix = self.config['macro_prefix']
 476         podomain = self.config['podomain']
 477         witness_c_macro = self.config['witness_c_macro']
 478         vc_files = self.config['vc_files']
 479         modules = [str(module) for module in moduletable['base']]
 480         avoids = [str(avoid) for avoid in moduletable['avoids']]
 481         emit += self.emiter.copyright_notice()
 482         emit += '''#
 483 # This file represents the specification of how gnulib-tool is used.
 484 # It acts as a cache: It is written and read by gnulib-tool.
 485 # In projects that use version control, this file is meant to be put under
 486 # version control, like the configure.ac and various Makefile.am files.
 487
 488
 489 # Specification in the form of a command-line invocation:
 490 #   %s
 491
 492 # Specification in the form of a few \
 493 gnulib-tool.m4 macro invocations:\n''' % actioncmd
 494         if not localdir or localdir.startswith('/'):
 495             relative_localdir = localdir
 496         else:  # if localdir or not localdir.startswith('/')
 497             relative_localdir = constants.relativize(destdir, localdir)
 498         emit += 'gl_LOCAL_DIR([%s])\n' % relative_localdir
 499         emit += 'gl_MODULES([\n'
 500         emit += '  %s\n' % '\n  '.join(modules)
 501         emit += '])\n'
 502         if self.config.checkTestFlag(TESTS['obsolete']):
 503             emit += 'gl_WITH_OBSOLETE\n'
 504         if self.config.checkTestFlag(TESTS['cxx-tests']):
 505             emit += 'gl_WITH_CXX_TESTS\n'
 506         if self.config.checkTestFlag(TESTS['privileged-tests']):
 507             emit += 'gl_WITH_PRIVILEGED_TESTS\n'
 508         if self.config.checkTestFlag(TESTS['unportable-tests']):
 509             emit += 'gl_WITH_UNPORTABLE_TESTS\n'
 510         if self.config.checkTestFlag(TESTS['all-tests']):
 511             emit += 'gl_WITH_ALL_TESTS\n'
 512         emit += 'gl_AVOID([%s])\n' % ' '.join(avoids)
 513         emit += 'gl_SOURCE_BASE([%s])\n' % sourcebase
 514         emit += 'gl_M4_BASE([%s])\n' % m4base
 515         emit += 'gl_PO_BASE([%s])\n' % pobase
 516         emit += 'gl_DOC_BASE([%s])\n' % docbase
 517         emit += 'gl_TESTS_BASE([%s])\n' % testsbase
 518         if self.config.checkTestFlag(TESTS['tests']):
 519             emit += 'gl_WITH_TESTS\n'
 520         emit += 'gl_LIB([%s])\n' % libname
 521         if lgpl != False:
 522             if lgpl == True:
 523                 emit += 'gl_LGPL\n'
 524             else:  # if lgpl != True
 525                 emit += 'gl_LGPL([%d])\n' % lgpl
 526         emit += 'gl_MAKEFILE_NAME([%s])\n' % makefile
 527         if conddeps:
 528             emit += 'gl_CONDITIONAL_DEPENDENCIES\n'
 529         if libtool:
 530             emit += 'gl_LIBTOOL\n'
 531         emit += 'gl_MACRO_PREFIX([%s])\n' % macro_prefix
 532         emit += 'gl_PO_DOMAIN([%s])\n' % podomain
 533         emit += 'gl_WITNESS_C_MACRO([%s])\n' % witness_c_macro
 534         if vc_files:
 535             emit += 'gl_VC_FILES([%s])\n' % vc_files
 536         if type(emit) is bytes:
 537             emit = emit.decode(ENCS['default'])
 538         return(constants.nlconvert(emit))
 539
 540     def gnulib_comp(self, files):
 541         '''GLImport.gnulib_comp(files) -> string
 542
 543         Emit the contents of generated $m4base/gnulib-comp.m4 file.
 544         GLConfig: destdir, localdir, tests, sourcebase, m4base, pobase, docbase,
 545         testsbase, conddeps, libtool, macro_prefix, podomain, vc_files.'''
 546         emit = string()
 547         assistant = self.assistant
 548         moduletable = self.moduletable
 549         destdir = self.config['destdir']
 550         localdir = self.config['localdir']
 551         auxdir = self.config['auxdir']
 552         testflags = list(self.config['testflags'])
 553         sourcebase = self.config['sourcebase']
 554         m4base = self.config['m4base']
 555         pobase = self.config['pobase']
 556         docbase = self.config['docbase']
 557         testsbase = self.config['testsbase']
 558         lgpl = self.config['lgpl']
 559         libname = self.config['libname']
 560         makefile = self.config['makefile']
 561         conddeps = self.config['conddeps']
 562         libtool = self.config['libtool']
 563         macro_prefix = self.config['macro_prefix']
 564         podomain = self.config['podomain']
 565         witness_c_macro = self.config['witness_c_macro']
 566         configure_ac = self.config['configure_ac']
 567         vc_files = self.config['vc_files']
 568         libtests = self.config['libtests']
 569         modules = [str(module) for module in moduletable['base']]
 570         avoids = [str(avoid) for avoid in moduletable['avoids']]
 571         emit += '# DO NOT EDIT! GENERATED AUTOMATICALLY!\n'
 572         emit += self.emiter.copyright_notice()
 573         emit += '''#
 574 # This file represents the compiled summary of the specification in
 575 # gnulib-cache.m4. It lists the computed macro invocations that need
 576 # to be invoked from configure.ac.
 577 # In projects that use version control, this file can be treated like
 578 # other built files.
 579
 580
 581 # This macro should be invoked from %s, in the section
 582 # "Checks for programs", right after AC_PROG_CC, and certainly before
 583 # any checks for libraries, header files, types and library functions.
 584 AC_DEFUN([%s_EARLY],
 585 [
 586   m4_pattern_forbid([^gl_[A-Z]])dnl the gnulib macro namespace
 587   m4_pattern_allow([^gl_ES$])dnl a valid locale name
 588   m4_pattern_allow([^gl_LIBOBJS$])dnl a variable
 589   m4_pattern_allow([^gl_LTLIBOBJS$])dnl a variable
 590   AC_REQUIRE([gl_PROG_AR_RANLIB])\n''' % (configure_ac, macro_prefix)
 591         uses_subdirs = False
 592         for module in moduletable['main']:
 593             # Test whether there are some source files in subdirectories.
 594             for file in module.getFiles():
 595                 if file.startswith('lib/') and file.endswith('.c') and \
 596                         file.count('/') > 1:
 597                     uses_subdirs = True
 598                     break
 599         if uses_subdirs:
 600             emit += '  AC_REQUIRE([AM_PROG_CC_C_O])\n'
 601         for module in moduletable['final']:
 602             emit += '  # Code from module %s:\n' % str(module)
 603             snippet = module.getAutoconfSnippet_Early()
 604             lines = [line for line in snippet.split(
 605                 constants.NL) if line != '']
 606             if lines:
 607                 emit += '  %s\n' % '\n  '.join(lines)
 608         emit += '])\n'
 609         emit += '''
 610 # This macro should be invoked from %s, in the section
 611 # "Check for header files, types and library functions".
 612 AC_DEFUN([%s_INIT],
 613 [\n''' % (configure_ac, macro_prefix)
 614         if libtool:
 615             emit += '  AM_CONDITIONAL([GL_COND_LIBTOOL], [true])\n'
 616             emit += '  gl_cond_libtool=true\n'
 617         else:  # if not libtool
 618             emit += '  AM_CONDITIONAL([GL_COND_LIBTOOL], [false])\n'
 619             emit += '  gl_cond_libtool=false\n'
 620             emit += '  gl_libdeps=\n'
 621             emit += '  gl_ltlibdeps=\n'
 622         replace_auxdir = False
 623         if auxdir != 'build-aux':
 624             replace_auxdir = True
 625         emit += '  gl_m4_base=\'%s\'\n' % m4base
 626         emit += self.emiter.initmacro_start(macro_prefix)
 627         emit += '  gl_source_base=\'%s\'\n' % sourcebase
 628         if witness_c_macro:
 629             emit += '  m4_pushdef([gl_MODULE_INDICATOR_CONDITION], [%s])\n' % \
 630                 witness_c_macro
 631         # Emit main autoconf snippets.
 632         emit += self.emiter.autoconfSnippets(moduletable['main'],
 633                                              moduletable, assistant, 0, True, False, True, replace_auxdir)
 634         if witness_c_macro:
 635             emit += '  m4_popdef([gl_MODULE_INDICATOR_CONDITION])\n'
 636         emit += '  # End of code from modules\n'
 637         emit += self.emiter.initmacro_end(macro_prefix)
 638         emit += '  gltests_libdeps=\n'
 639         emit += '  gltests_ltlibdeps=\n'
 640         emit += self.emiter.initmacro_start('%stests' % macro_prefix)
 641         emit += '  gl_source_base=\'%s\'\n' % testsbase
 642         # Define a tests witness macro that depends on the package.
 643         # PACKAGE is defined by AM_INIT_AUTOMAKE, PACKAGE_TARNAME is defined by
 644         # AC_INIT.
 645         # See <https://lists.gnu.org/r/automake/2009-05/msg00145.html>.
 646         emit += 'changequote(,)dnl\n'
 647         emit += '  %stests_WITNESS=' % macro_prefix
 648         emit += 'IN_`echo "${PACKAGE-$PACKAGE_TARNAME}" | LC_ALL=C tr \
 649 abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ | LC_ALL=C sed -e \
 650 \'s/[^A-Z0-9_]/_/g\'`_GNULIB_TESTS\n'
 651         emit += 'changequote([, ])dnl\n'
 652         emit += '  AC_SUBST([%stests_WITNESS])\n' % macro_prefix
 653         emit += '  gl_module_indicator_condition=$%stests_WITNESS\n' % macro_prefix
 654         emit += '  m4_pushdef([gl_MODULE_INDICATOR_CONDITION], '
 655         emit += '[$gl_module_indicator_condition])\n'
 656         # Emit tests autoconf snippets.
 657         emit += self.emiter.autoconfSnippets(moduletable['tests'],
 658                                              moduletable, assistant, 0, True, True, True, replace_auxdir)
 659         emit += '  m4_popdef([gl_MODULE_INDICATOR_CONDITION])\n'
 660         emit += self.emiter.initmacro_end('%stests' % macro_prefix)
 661         # _LIBDEPS and _LTLIBDEPS variables are not needed if this library is
 662         # created using libtool, because libtool already handles the dependencies.
 663         if not libtool:
 664             libname_upper = libname.upper().replace('-', '_')
 665             emit += '  %s_LIBDEPS="$gl_libdeps"\n' % libname_upper
 666             emit += '  AC_SUBST([%s_LIBDEPS])\n' % libname_upper
 667             emit += '  %s_LTLIBDEPS="$gl_ltlibdeps"\n' % libname_upper
 668             emit += '  AC_SUBST([%s_LTLIBDEPS])\n' % libname_upper
 669         if libtests:
 670             emit += '  LIBTESTS_LIBDEPS="$gltests_libdeps"\n'
 671             emit += '  AC_SUBST([LIBTESTS_LIBDEPS])\n'
 672         emit += '])\n'
 673         emit += self.emiter.initmacro_done(macro_prefix, sourcebase)
 674         emit += self.emiter.initmacro_done('%stests' % macro_prefix, testsbase)
 675         emit += '''
 676 # This macro records the list of files which have been installed by
 677 # gnulib-tool and may be removed by future gnulib-tool invocations.
 678 AC_DEFUN([%s_FILE_LIST], [\n''' % macro_prefix
 679         emit += '  %s\n' % '\n  '.join(files)
 680         emit += '])\n'
 681         if type(emit) is bytes:
 682             emit = emit.decode(ENCS['default'])
 683         return(emit)
 684
 685     def _done_dir_(self, directory, dirs_added, dirs_removed):
 686         '''GLImport._done_dir_(directory, dirs_added, dirs_removed)
 687
 688         This method is used to determine ignore argument for _update_ignorelist_
 689         method and then call it.'''
 690         destdir = self.config['destdir']
 691         if isdir(joinpath(destdir, 'CVS')) or \
 692                 isdir(joinpath(destdir, directory, 'CVS')) or \
 693                 isfile(joinpath(destdir, directory, '.cvsignore')):
 694             self._update_ignorelist_(directory, '.cvsignore',
 695                                      dirs_added, dirs_removed)
 696         if isdir(joinpath(destdir, '.git')) or \
 697                 isfile(joinpath(destdir, directory, '.gitignore')):
 698             self._update_ignorelist_(directory, '.gitignore',
 699                                      dirs_added, dirs_removed)
 700
 701     def _update_ignorelist_(self, directory, ignore, dirs_added, dirs_removed):
 702         '''GLImport._update_ignorelist_(directory, ignore, dirs_added, dirs_removed)
 703
 704         Update .gitignore or .cvsignore files.'''
 705         result = string()
 706         destdir = self.config['destdir']
 707         if ignore == '.gitignore':
 708             anchor = '/'
 709         else:
 710             anchor = ''
 711         srcpath = joinpath(destdir, directory, ignore)
 712         backupname = '%s~' % srcpath
 713         if isfile(srcpath):
 714             if dirs_added or dirs_removed:
 715                 with codecs.open(srcpath, 'rb', 'UTF-8') as file:
 716                     srcdata = file.read()
 717                 dirs_ignore = sorted(set(srcdata.split('\n')))
 718                 dirs_ignore = [line for line in dirs_ignore if line.strip()]
 719                 srcdata = '\n'.join(sorted(set(dirs_ignore))).strip()
 720                 dirs_ignore += [d for d in dirs_added if d not in dirs_ignore]
 721                 dirs_ignore = [d for d in dirs_ignore if d in dirs_removed]
 722                 dirs_ignore = ['%s%s' % (anchor, d) for d in dirs_ignore]
 723                 dirs_ignore = sorted(set(dirs_ignore))
 724                 destdata = '\n'.join(sorted(set(dirs_ignore))).strip()
 725                 if srcdata != destdata:
 726                     if not self.config['dryrun']:
 727                         print('Updating %s (backup in %s)' %
 728                               (srcpath, backupname))
 729                         shutil.copy2(srcpath, backupname)
 730                         result = string()
 731                         with codecs.open(srcpath, 'ab', 'UTF-8') as file:
 732                             file.write(destdata)
 733                     else:  # if self.config['dryrun']
 734                         print('Updating %s (backup in %s)' %
 735                               (srcpath, backupname))
 736         else:  # if not isfile(srcpath)
 737             if dirs_added:
 738                 if not self.config['dryrun']:
 739                     print('Creating %s' % srcpath)
 740                     dirs_added = sorted(set(dirs_added))
 741                     dirs_added = ['%s%s' % (anchor, d) for d in dirs_added]
 742                     if ignore == '.cvsignore':
 743                         dirs_added = ['.deps', '.dirstamp'] + dirs_added
 744                     with codecs.open(srcpath, 'wb', 'UTF-8') as file:
 745                         file.write('\n'.join(dirs_added))
 746                         file.write('\n')
 747                 else:  # if self.config['dryrun']
 748                     print('Create %s' % srcpath)
 749
 750     def prepare(self):
 751         '''Make all preparations before the execution of the code.
 752         Returns filetable and sed transformers, which change the license.'''
 753         destdir = self.config['destdir']
 754         localdir = self.config['localdir']
 755         auxdir = self.config['auxdir']
 756         modules = list(self.config['modules'])
 757         avoids = list(self.config['avoids'])
 758         testflags = list(self.config['testflags'])
 759         sourcebase = self.config['sourcebase']
 760         m4base = self.config['m4base']
 761         pobase = self.config['pobase']
 762         docbase = self.config['docbase']
 763         testsbase = self.config['testsbase']
 764         lgpl = self.config['lgpl']
 765         copyrights = self.config['copyrights']
 766         libname = self.config['libname']
 767         makefile = self.config['makefile']
 768         conddeps = self.config['conddeps']
 769         libtool = self.config['libtool']
 770         macro_prefix = self.config['macro_prefix']
 771         podomain = self.config['podomain']
 772         witness_c_macro = self.config['witness_c_macro']
 773         vc_files = self.config['vc_files']
 774         configure_ac = self.config['configure_ac']
 775         ac_version = self.config['ac_version']
 776         verbose = self.config['verbosity']
 777         base_modules = sorted(
 778             set([self.modulesystem.find(m) for m in modules]))
 779         avoids = sorted(set([self.modulesystem.find(a) for a in avoids]))
 780
 781         # Perform transitive closure.
 782         self.moduletable.setAvoids(avoids)
 783         final_modules = self.moduletable.transitive_closure(base_modules)
 784
 785         # Show final module list.
 786         if verbose >= 0:
 787             bold_on = ''
 788             bold_off = ''
 789             term = os.getenv('TERM')
 790             if term == 'xterm':
 791                 bold_on = '\x1b[1m'
 792                 bold_off = '\x1b[0m'
 793             print('Module list with included dependencies (indented):')
 794             for module in final_modules:
 795                 if str(module) in self.config.getModules():
 796                     print('  %s%s%s' % (bold_on, module, bold_off))
 797                 else:  # if str(module) not in self.config.getModules()
 798                     print('    %s' % module)
 799
 800         # Separate modules into main_modules and tests_modules.
 801         modules = self.moduletable.transitive_closure_separately(
 802             base_modules, final_modules)
 803         main_modules, tests_modules = modules
 804
 805         # Transmit base_modules, final_modules, main_modules and tests_modules.
 806         self.moduletable.setBaseModules(base_modules)
 807         self.moduletable.setFinalModules(final_modules)
 808         self.moduletable.setMainModules(main_modules)
 809         self.moduletable.setTestsModules(tests_modules)
 810
 811         # Print main_modules and tests_modules.
 812         if verbose >= 1:
 813             print('Main module list:')
 814             for module in main_modules:
 815                 print('  %s' % str(module))
 816             print('Tests-related module list:')
 817             for module in tests_modules:
 818                 print('  %s' % str(module))
 819
 820         # Determine whether a $testsbase/libtests.a is needed.
 821         libtests = False
 822         for module in tests_modules:
 823             files = module.getFiles()
 824             for file in files:
 825                 if file.startswith('lib/'):
 826                     libtests = True
 827                     break
 828         if libtests:
 829             self.config.enableLibtests()
 830
 831         # Add dummy package if it is needed.
 832         main_modules = self.moduletable.add_dummy(main_modules)
 833         if libtests:  # if we need to use libtests.a
 834             tests_modules = self.moduletable.add_dummy(tests_modules)
 835
 836         # Check license incompatibilities.
 837         listing = list()
 838         compatibilities = dict()
 839         incompatibilities = string()
 840         compatibilities['all'] = ['GPLed build tool', 'public domain', 'unlimited',
 841                                   'unmodifiable license text']
 842         compatibilities[3] = ['LGPL', 'LGPLv2+', 'LGPLv3+']
 843         compatibilities[2] = ['LGPLv2+']
 844         if lgpl:
 845             for module in main_modules:
 846                 license = module.getLicense()
 847                 if license not in compatibilities['all']:
 848                     if lgpl == 3 or lgpl == True:
 849                         if license not in compatibilities[3]:
 850                             listing.append(tuple([str(module), license]))
 851                     elif lgpl == 2:
 852                         if license not in compatibilities[2]:
 853                             listing.append(tuple([str(module), license]))
 854             if listing:
 855                 raise(GLError(11, listing))
 856
 857         # Print notices from modules.
 858         for module in main_modules:
 859             notice = module.getNotice()
 860             notice = notice.strip()
 861             if notice:
 862                 print('Notice from module %s:' % str(module))
 863                 pattern = compiler('^(.*?)$', re.S | re.M)
 864                 notice = pattern.sub('  \\1', notice)
 865                 print(notice)
 866
 867         # Determine script to apply to imported library files.
 868         lgpl2gpl = '''
 869       s/GNU Lesser General/GNU General/g
 870       s/Lesser General Public License/General Public License/g
 871       s/GNU Library General/GNU General/g
 872       s/Library General Public License/General Public License/g
 873       s/version 2\\(.1\\)\\{0,1\\}\\([ ,]\\)/version 3\\2/g'''
 874         sed_transform_lib_file = string()
 875         if 'config-h' in [str(module) for module in main_modules]:
 876             sed_transform_lib_file += '''
 877         s/^#ifdef[\t ]*HAVE_CONFIG_H[\t ]*$/#if 1/
 878       '''
 879         sed_transform_main_lib_file = sed_transform_lib_file
 880         if copyrights:
 881             if lgpl:  # if lgpl is enabled
 882                 if lgpl == 3:
 883                     sed_transform_main_lib_file += '''
 884             s/GNU General/GNU Lesser General/g
 885             s/General Public License/Lesser General Public License/g
 886             s/Lesser Lesser General Public License/Lesser General Public''' \
 887                         + ' License/g'
 888                 elif lgpl == 2:
 889                     sed_transform_main_lib_file += '''
 890             s/GNU General/GNU Lesser General/g
 891             s/General Public License/Lesser General Public License/g
 892             s/Lesser Lesser General Public License/Lesser General Public''' \
 893                         + '''License/g
 894             s/version [23]\\([ ,]\\)/version 2.1\\1/g'''
 895             else:  # if lgpl is disabled
 896                 sed_transform_main_lib_file += lgpl2gpl
 897
 898         # Determine script to apply to auxiliary files that go into $auxdir/.
 899         sed_transform_build_aux_file = string()
 900         if copyrights:
 901             sed_transform_build_aux_file += lgpl2gpl
 902
 903         # Determine script to apply to library files that go into $testsbase/.
 904         sed_transform_testsrelated_lib_file = sed_transform_lib_file
 905         if copyrights:
 906             sed_transform_testsrelated_lib_file += lgpl2gpl
 907
 908         # Determine the final file lists.
 909         main_filelist, tests_filelist = \
 910             self.moduletable.filelist_separately(main_modules, tests_modules)
 911         filelist = sorted(
 912             set(main_filelist + tests_filelist), key=string.lower)
 913         if not filelist:
 914             raise(GLError(12, None))
 915
 916         # Print list of files.
 917         if verbose >= 0:
 918             print('File list:')
 919             for file in filelist:
 920                 if file.startswith('tests=lib/'):
 921                     rest = file[10:]
 922                     print('  lib/%s -> tests/%s' % (rest, rest))
 923                 else:
 924                     print('  %s' % file)
 925
 926         # Prepare basic filelist and basic old_files/new_files variables.
 927         filelist = sorted(set(filelist))
 928         new_files = filelist + ['m4/gnulib-tool.m4']
 929         old_files = list(self.cache['files'])
 930         path = joinpath(destdir, m4base, 'gnulib-tool.m4')
 931         if isfile(path):
 932             old_files += [joinpath('m4', 'gnulib-tool.m4')]
 933
 934         # Construct tables and transformers.
 935         transformers = dict()
 936         transformers['lib'] = string(sed_transform_lib_file)
 937         transformers['aux'] = string(sed_transform_build_aux_file)
 938         transformers['main'] = string(sed_transform_main_lib_file)
 939         transformers['tests'] = string(sed_transform_testsrelated_lib_file)
 940         old_table = list()
 941         new_table = list()
 942         for src in old_files:
 943             dest = self.rewrite_old_files([src])[-1]
 944             old_table += [tuple([dest, src])]
 945         for src in new_files:
 946             dest = self.rewrite_new_files([src])[-1]
 947             new_table += [tuple([dest, src])]
 948         old_table = sorted(set(old_table))
 949         new_table = sorted(set(new_table))
 950
 951         # Prepare the filetable.
 952         filetable = dict()
 953         filetable['all'] = sorted(set(filelist))
 954         filetable['old'] = \
 955             sorted(set(old_table), key=lambda t: tuple(t[0].lower()))
 956         filetable['new'] = \
 957             sorted(set(new_table), key=lambda t: tuple(t[0].lower()))
 958         filetable['added'] = list()
 959         filetable['removed'] = list()
 960
 961         # Return the result.
 962         result = tuple([filetable, transformers])
 963         return(result)
 964
 965     def execute(self, filetable, transformers):
 966         '''Perform operations on the lists of files, which are given in a special
 967         format except filelist argument. Such lists of files can be created using
 968         GLImport.prepare() function.'''
 969         if type(filetable) is not dict:
 970             raise(TypeError('filetable must be a dict, not %s' %
 971                             type(filetable).__name__))
 972         for key in ['all', 'old', 'new', 'added', 'removed']:
 973             if key not in filetable:
 974                 raise(KeyError('filetable must contain key %s' % repr(key)))
 975         destdir = self.config['destdir']
 976         localdir = self.config['localdir']
 977         auxdir = self.config['auxdir']
 978         modules = list(self.config['modules'])
 979         avoids = list(self.config['avoids'])
 980         testflags = list(self.config['testflags'])
 981         sourcebase = self.config['sourcebase']
 982         m4base = self.config['m4base']
 983         pobase = self.config['pobase']
 984         docbase = self.config['docbase']
 985         testsbase = self.config['testsbase']
 986         lgpl = self.config['lgpl']
 987         copyrights = self.config['copyrights']
 988         libname = self.config['libname']
 989         makefile = self.config['makefile']
 990         conddeps = self.config['conddeps']
 991         libtool = self.config['libtool']
 992         macro_prefix = self.config['macro_prefix']
 993         podomain = self.config['podomain']
 994         witness_c_macro = self.config['witness_c_macro']
 995         vc_files = self.config['vc_files']
 996         configure_ac = self.config['configure_ac']
 997         ac_version = self.config['ac_version']
 998         verbose = self.config['verbosity']
 999         actioncmd = self.actioncmd()
1000
1001         # Create all necessary directories.
1002         dirs = list()
1003         if pobase:
1004             dirs += [pobase]
1005         if [file for file in filetable['all'] if file.startswith('doc/')]:
1006             dirs += [docbase]
1007         dirs += [sourcebase, m4base, auxdir]
1008         dirs += [os.path.dirname(pair[0]) for pair in filetable['new']]
1009         dirs = sorted(set([joinpath(destdir, d) for d in dirs]))
1010         for directory in dirs:
1011             if not isdir(directory):
1012                 print('Creating directory %s' % directory)
1013                 if not self.config['dryrun']:
1014                     try:  # Try to create directory
1015                         os.makedirs(directory)
1016                     except Exception as error:
1017                         raise(GLError(13, directory))
1018                 else:  # if self.config['dryrun']
1019                     print('Create directory %s' % directory)
1020
1021         # Create GLFileAssistant instance to process files.
1022         self.assistant = GLFileAssistant(self.config, transformers)
1023
1024         # Files which are in filetable['old'] and not in filetable['new'].
1025         # They will be removed and added to filetable['removed'] list.
1026         pairs = [f for f in filetable['old'] if f not in filetable['old']]
1027         pairs = sorted(set(pairs), key=lambda t: tuple(t[0].lower()))
1028         files = sorted(set(pair[0] for pair in pairs))
1029         for file in files:
1030             path = joinpath(destdir, file)
1031             if isfile(path) or os.path.islink(path):
1032                 if not self.config['dryrun']:
1033                     backup = string('%s~' % path)
1034                     print('Removing file %s (backup in )' % (path, backup))
1035                     try:  # Try to move file
1036                         if os.path.exists(backup):
1037                             os.remove(backup)
1038                         shutil.move(path, '%s~' % path)
1039                     except Exception as error:
1040                         raise(GLError(14, file))
1041                 else:  # if self.config['dryrun']
1042                     print('Remove file %s (backup in %s~)' % (path, path))
1043                 filetable['removed'] += [file]
1044
1045         # Files which are in filetable['new'] and not in filetable['old'].
1046         # They will be added/updated and added to filetable['added'] list.
1047         already_present = False
1048         pairs = [f for f in filetable['new'] if f not in filetable['old']]
1049         pairs = sorted(set(pairs))
1050         for pair in pairs:
1051             original = pair[1]
1052             rewritten = pair[0]
1053             self.assistant.setOriginal(original)
1054             self.assistant.setRewritten(rewritten)
1055             self.assistant.add_or_update(already_present)
1056
1057         # Files which are in filetable['new'] and in filetable['old'].
1058         # They will be added/updated and added to filetable['added'] list.
1059         already_present = True
1060         pairs = [f for f in filetable['new'] if f in filetable['old']]
1061         pairs = sorted(set(pairs))
1062         for pair in pairs:
1063             original = pair[1]
1064             rewritten = pair[0]
1065             self.assistant.setOriginal(original)
1066             self.assistant.setRewritten(rewritten)
1067             self.assistant.add_or_update(already_present)
1068
1069         # Add files which were added to the list of filetable['added'].
1070         filetable['added'] += self.assistant.getFiles()
1071         filetable['added'] = sorted(set(filetable['added']))
1072
1073         # Determine include_guard_prefix.
1074         include_guard_prefix = self.config['include_guard_prefix']
1075
1076         # Determine makefile name.
1077         if not makefile:
1078             makefile_am = string('Makefile.am')
1079         else:  # if makefile
1080             makefile_am = makefile
1081
1082         # Create normal Makefile.ams.
1083         for_test = False
1084
1085         # Setup list of Makefile.am edits that are to be performed afterwards.
1086         # Some of these edits apply to files that we will generate; others are
1087         # under the responsibility of the developer.
1088         makefile_am_edits = dict()
1089         if makefile_am == 'Makefile.am':
1090             sourcebase_dir = os.path.dirname(sourcebase)
1091             sourcebase_base = os.path.basename(sourcebase)
1092             self.makefiletable.editor(
1093                 sourcebase_dir, 'SUBDIRS', sourcebase_base)
1094         if pobase:
1095             pobase_dir = os.path.dirname(pobase)
1096             pobase_base = os.path.basename(pobase)
1097             self.makefiletable.editor(pobase_dir, 'SUBDIRS', pobase_base)
1098         if self.config.checkTestFlag(TESTS['tests']):
1099             if makefile_am == 'Makefile.am':
1100                 testsbase_dir = os.path.dirname(testsbase)
1101                 testsbase_base = os.path.basename(testsbase)
1102                 self.makefiletable.editor(
1103                     testsbase_dir, 'SUBDIRS', testsbase_base)
1104         self.makefiletable.editor('', 'ACLOCAL_AMFLAGS', '-I %s' % m4base)
1105         self.makefiletable.parent()
1106
1107         # Create library makefile.
1108         basename = joinpath(sourcebase, makefile_am)
1109         tmpfile = self.assistant.tmpfilename(basename)
1110         emit, uses_subdirs = self.emiter.lib_Makefile_am(basename,
1111                                                          self.moduletable['main'], self.moduletable, self.makefiletable,
1112                                                          actioncmd, for_test)
1113         with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
1114             file.write(emit)
1115         filename, backup, flag = self.assistant.super_update(basename, tmpfile)
1116         if flag == 1:
1117             if not self.config['dryrun']:
1118                 print('Updating %s (backup in %s)' % (filename, backup))
1119             else:  # if self.config['dryrun']
1120                 print('Update %s (backup in %s)' % (filename, backup))
1121         elif flag == 2:
1122             if not self.config['dryrun']:
1123                 print('Creating %s' % filename)
1124             else:  # if self.config['dryrun']:
1125                 print('Create %s' % filename)
1126             filetable['added'] += [filename]
1127         if isfile(tmpfile):
1128             os.remove(tmpfile)
1129
1130         # Create po/ directory.
1131         filesystem = GLFileSystem(self.config)
1132         if pobase:
1133             # Create po makefile and auxiliary files.
1134             for file in ['Makefile.in.in', 'remove-potcdate.sin']:
1135                 tmpfile = self.assistant.tmpfilename(joinpath(pobase, file))
1136                 path = joinpath('build-aux', 'po', file)
1137                 lookedup, flag = filesystem.lookup(path)
1138                 shutil.move(lookedup, tmpfile)
1139                 basename = joinpath(pobase, file)
1140                 filename, backup, flag = self.assistant.super_update(
1141                     basename, tmpfile)
1142                 if flag == 1:
1143                     if not self.config['dryrun']:
1144                         print('Updating %s (backup in %s)' %
1145                               (filename, backup))
1146                     else:  # if self.config['dryrun']
1147                         print('Update %s (backup in %s)' % (filename, backup))
1148                 elif flag == 2:
1149                     if not self.config['dryrun']:
1150                         print('Creating %s' % filename)
1151                     else:  # if self.config['dryrun']:
1152                         print('Create %s' % filename)
1153                     filetable['added'] += [filename]
1154             if isfile(tmpfile):
1155                 os.remove(tmpfile)
1156
1157             # Create po makefile parameterization, part 1.
1158             basename = joinpath(pobase, 'Makevars')
1159             tmpfile = self.assistant.tmpfilename(basename)
1160             emit = self.emiter.po_Makevars()
1161             with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
1162                 file.write(emit)
1163             filename, backup, flag = self.assistant.super_update(
1164                 basename, tmpfile)
1165             if flag == 1:
1166                 if not self.config['dryrun']:
1167                     print('Updating %s (backup in %s)' % (filename, backup))
1168                 else:  # if self.config['dryrun']
1169                     print('Update %s (backup in %s)' % (filename, backup))
1170             elif flag == 2:
1171                 if not self.config['dryrun']:
1172                     print('Creating %s' % filename)
1173                 else:  # if self.config['dryrun']:
1174                     print('Create %s' % filename)
1175                 filetable['added'] += [filename]
1176             if isfile(tmpfile):
1177                 os.remove(tmpfile)
1178
1179             # Create po makefile parameterization, part 2.
1180             basename = joinpath(pobase, 'POTFILES.in')
1181             tmpfile = self.assistant.tmpfilename(basename)
1182             with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
1183                 file.write(self.emiter.po_POTFILES_in(filetable['all']))
1184             basename = joinpath(pobase, 'POTFILES.in')
1185             filename, backup, flag = self.assistant.super_update(
1186                 basename, tmpfile)
1187             if flag == 1:
1188                 if not self.config['dryrun']:
1189                     print('Updating %s (backup in %s)' % (filename, backup))
1190                 else:  # if self.config['dryrun']
1191                     print('Update %s (backup in %s)' % (filename, backup))
1192             elif flag == 2:
1193                 if not self.config['dryrun']:
1194                     print('Creating %s' % filename)
1195                 else:  # if self.config['dryrun']:
1196                     print('Create %s' % filename)
1197                 filetable['added'] += [filename]
1198             if isfile(tmpfile):
1199                 os.remove(tmpfile)
1200
1201             # Fetch PO files.
1202             TP_URL = 'https://translationproject.org/latest/'
1203             if not self.config['dryrun']:
1204                 print('Fetching gnulib PO files from %s' % TP_URL)
1205                 os.chdir(joinpath(destdir, pobase))
1206                 args = ['wget', '--no-verbose', '--mirror', '--level=1', '-nd', '-A.po', '-P', '.',
1207                         '%sgnulib/' % TP_URL]
1208                 sp.call(args, shell=True)
1209             else:  # if self.config['dryrun']
1210                 print('Fetch gnulib PO files from %s' % TP_URL)
1211
1212             # Create po/LINGUAS.
1213             basename = joinpath(pobase, 'LINGUAS')
1214             if not self.config['dryrun']:
1215                 tmpfile = self.assistant.tmpfilename(basename)
1216                 data = string('# Set of available languages.\n')
1217                 files = [constants.subend('.po', '', file)
1218                          for file in os.listdir(joinpath(destdir, pobase))]
1219                 files = [file.decode(ENCS['default']) if type(file) is bytes
1220                          else file for file in files]
1221                 data += '\n'.join(files)
1222                 with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
1223                     file.write(data)
1224                 filename, backup, flag = self.assistant.super_update(
1225                     basename, tmpfile)
1226                 if flag == 1:
1227                     print('Updating %s (backup in %s)' % (filename, backup))
1228                 elif flag == 2:
1229                     print('Creating %s' % filename)
1230                     filetable['added'] += [filename]
1231                 if isfile(tmpfile):
1232                     os.remove(tmpfile)
1233             else:  # if not self.config['dryrun']
1234                 backupname = '%s~' % basename
1235                 if isfile(destdir, basename):
1236                     print('Update %s (backup in %s)' % (basename, backupname))
1237                 else:  # if not isfile(destdir, basename)
1238                     print('Create %s' % basename)
1239
1240         # Create m4/gnulib-cache.m4.
1241         basename = joinpath(m4base, 'gnulib-cache.m4')
1242         tmpfile = self.assistant.tmpfilename(basename)
1243         emit = self.gnulib_cache()
1244         with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
1245             file.write(emit)
1246         filename, backup, flag = self.assistant.super_update(basename, tmpfile)
1247         if flag == 1:
1248             if not self.config['dryrun']:
1249                 print('Updating %s (backup in %s)' % (filename, backup))
1250             else:  # if self.config['dryrun']
1251                 print('Update %s (backup in %s)' % (filename, backup))
1252         elif flag == 2:
1253             if not self.config['dryrun']:
1254                 print('Creating %s' % filename)
1255             else:  # if self.config['dryrun']:
1256                 print('Create %s' % filename)
1257                 if emit[-2:] == '\r\n':
1258                     emit = emit[:-2]
1259                 elif emit[-1:] == '\n':
1260                     emit = emit[:-1]
1261                 print(emit)
1262         if isfile(tmpfile):
1263             os.remove(tmpfile)
1264
1265         # Create m4/gnulib-comp.m4.
1266         basename = joinpath(m4base, 'gnulib-comp.m4')
1267         tmpfile = self.assistant.tmpfilename(basename)
1268         emit = self.gnulib_comp(filetable['all'])
1269         with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
1270             file.write(emit)
1271         filename, backup, flag = self.assistant.super_update(basename, tmpfile)
1272         if flag == 1:
1273             if not self.config['dryrun']:
1274                 print('Updating %s (backup in %s)' % (filename, backup))
1275             else:  # if self.config['dryrun']
1276                 print('Update %s (backup in %s)' % (filename, backup))
1277         elif flag == 2:
1278             if not self.config['dryrun']:
1279                 print('Creating %s' % filename)
1280             else:  # if self.config['dryrun']:
1281                 print('Create %s' % filename)
1282                 if emit[-2:] == '\r\n':
1283                     emit = emit[:-2]
1284                 elif emit[-1:] == '\n':
1285                     emit = emit[:-1]
1286                 print(emit)
1287         if isfile(tmpfile):
1288             os.remove(tmpfile)
1289
1290         # Create tests Makefile.
1291         inctests = self.config.checkTestFlag(TESTS['tests'])
1292         if inctests:
1293             basename = joinpath(testsbase, makefile_am)
1294             tmpfile = self.assistant.tmpfilename(basename)
1295             emit, uses_subdirs = self.emiter.lib_Makefile_am(basename,
1296                                                              self.moduletable['tests'], self.moduletable, self.makefiletable,
1297                                                              actioncmd, for_test)
1298             with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
1299                 file.write(emit)
1300             filename, backup, flag = self.assistant.super_update(
1301                 basename, tmpfile)
1302             if flag == 1:
1303                 if not self.config['dryrun']:
1304                     print('Updating %s (backup in %s)' % (filename, backup))
1305                 else:  # if self.config['dryrun']
1306                     print('Update %s (backup in %s)' % (filename, backup))
1307             elif flag == 2:
1308                 if not self.config['dryrun']:
1309                     print('Creating %s' % filename)
1310                 else:  # if self.config['dryrun']:
1311                     print('Create %s' % filename)
1312                 filetable['added'] += [filename]
1313             if isfile(tmpfile):
1314                 os.remove(tmpfile)
1315
1316         # Update the .cvsignore and .gitignore files.
1317         ignorelist = list()
1318         filetable['added'] = sorted(set(filetable['added']))
1319         filetable['removed'] = sorted(set(filetable['added']))
1320         for file in filetable['added']:
1321             directory, basename = os.path.split(file)
1322             ignorelist += [tuple([directory, '|A|', basename])]
1323         for file in filetable['removed']:
1324             directory, basename = os.path.split(file)
1325             ignorelist += [tuple([directory, '|R|', basename])]
1326         last_dir = string()
1327         last_dirs_added = list()
1328         last_dirs_removed = list()
1329         for row in ignorelist:
1330             next_dir = row[0]
1331             operand = row[1]
1332             filename = row[2]
1333             if next_dir != last_dir:
1334                 self._done_dir_(last_dir, last_dirs_added, last_dirs_removed)
1335                 last_dir = next_dir
1336                 last_dirs_added = list()
1337                 last_dirs_removed = list()
1338             if operand == '|A|':
1339                 last_dirs_added += [filename]
1340             elif operand == '|R|':
1341                 last_dirs_removed += [filename]
1342         self._done_dir_(last_dir, last_dirs_added, last_dirs_removed)
1343         exit()
1344
1345         # Finish the work.
1346         print('Finished.\n')
1347         print('You may need to add #include directives \
1348 for the following .h files.')
1349         modules = sorted(set([module for module in self.moduletable['base']
1350                               if module in self.moduletable['main']]))
1351         # First the #include <...> directives without #ifs, sorted for convenience,
1352         # then the #include "..." directives without #ifs, sorted for convenience,
1353         # then the #include directives that are surrounded by #ifs. Not sorted.
1354         includes_angles = list()
1355         includes_quotes = list()
1356         includes_if = list()
1357         for module in modules:
1358             include = module.getInclude()
1359             for include in include.split('\n'):
1360                 if '%s#if' % constants.NL in '%s%s' % (constants.NL, include):
1361                     includes_if += [include]
1362                 # if '%s#if' % constants.NL in '%s%s' % (constants.NL, include)
1363                 else:
1364                     if 'include "' in include:
1365                         includes_quotes += [include]
1366                     else:  # if 'include "' not in include
1367                         includes_angles += [include]
1368         includes_angles = sorted(set(includes_angles))
1369         includes_quotes = sorted(set(includes_quotes))
1370         includes = includes_angles + includes_quotes + includes_if
1371         includes = [include for include in includes if include.split()]
1372         for include in includes:
1373             print('  %s' % include)
1374
1375         # Get link directives.
1376         links = [module.getLink() for module in self.moduletable['main']]
1377         links = sorted(set([link for link in links if link.strip()]))
1378         if links:
1379             print('''
1380 You may need to use the following Makefile variables when linking.
1381 Use them in <program>_LDADD when linking a program, or
1382 in <library>_a_LDFLAGS or <library>_la_LDFLAGS when linking a library.''')
1383             for link in links:
1384                 print('  %s' % link)
1385
1386         # Print reminders.
1387         print('')
1388         print('Don\'t forget to')
1389         if makefile_am == 'Makefile.am':
1390             print('  - add "%s/Makefile" to AC_CONFIG_FILES in %s,' %
1391                   (sourcebase, configure_ac))
1392         else:  # if makefile_am != 'Makefile.am'
1393             print('  - "include %s" from within "%s/Makefile.am",' %
1394                   (makefile, sourcebase))
1395         if pobase:
1396             print('  - add "%s/Makefile.in to AC_CONFIG_FILES in %s,' %
1397                   (pobase, configure_ac))
1398         if inctests:
1399             if makefile_am == 'Makefile.am':
1400                 print('  - add "%s/Makefile" to AC_CONFIG_FILES in %s,' %
1401                       (testsbase, configure_ac))
1402             else:  # if makefile_am != 'Makefile.am'
1403                 print('  - "include %s" from within "%s/Makefile.am",' %
1404                       (makefile, testsbase))
1405         # Print makefile edits.
1406         current_edit = int()
1407         makefile_am_edits = self.makefiletable.count()
1408         while current_edit != makefile_am_edits:
1409             dictionary = self.makefiletable[current_edit]
1410             if dictionary['var']:
1411                 print('  - mention "%s" in %s in %s,' %
1412                       (dictionary['val'], dictionary['var'],
1413                        joinpath(dictionary['dir'], 'Makefile.am')))
1414             current_edit += 1
1415
1416         # Detect position_early_after.
1417         with codecs.open(configure_ac, 'rb', 'UTF-8') as file:
1418             data = file.read()
1419         match_result1 = \
1420             bool(compiler('^ *AC_PROG_CC_STDC', re.S | re.M).findall(data))
1421         match_result2 = \
1422             bool(compiler('^ *AC_PROG_CC_C99', re.S | re.M).findall(data))
1423         if match_result1:
1424             position_early_after = 'AC_PROG_CC_STDC'
1425         elif match_result2:
1426             position_early_after = 'AC_PROG_CC_C99'
1427         else:  # if not any([match_result1, match_result2])
1428             position_early_after = 'AC_PROG_CC'
1429         print('  - invoke %s_EARLY in %s, right after %s,' %
1430               (macro_prefix, configure_ac, position_early_after))
1431         print('  - invoke %s_INIT in %s.' %
1432               (macro_prefix, configure_ac))
1433         sp.call(['rm', '-rf', self.config['tempdir']], shell=False)