gnulib-tool.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (C) 2002-2024 Free Software Foundation, Inc.
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  17 #
  18
  19 # This program is meant for authors or maintainers which want to import
  20 # modules from gnulib into their packages.
  21
  22 # CODING STYLE for this file and its companions:
  23 # Like PEP 8 <https://peps.python.org/pep-0008/>, except
  24 # - Line length is not limited to 79 characters.
  25 # - Line breaking before or after binary operators? Better before, like in GNU.
  26 # - Place line breaks to help reading the code:
  27 #   Avoid line breaks inside expressions, if they can be avoided.
  28 #   Do use line breaks to separate the parts of [ ... for ... ] iterations.
  29 # You can use this command to check the style:
  30 #   $ pycodestyle --max-line-length=136 --ignore=E265,W503,E241,E711,E712,E201,E202,E221 gnulib-tool.py pygnulib/*.py
  31
  32 # You can use this command to check for mistakes:
  33 #   $ pylint --disable=C0103,C0114,C0121,C0209,C0301,C0302,R0902,R0912,R0913,R0914,R0915,R1705,R1702,R1720 gnulib-tool.py pygnulib/*.py
  34
  35
  36 #===============================================================================
  37 # Define global imports
  38 #===============================================================================
  39 import os
  40 import re
  41 import sys
  42 import codecs
  43 import random
  44 import argparse
  45 import subprocess as sp
  46 import shlex
  47 from tempfile import mktemp
  48 from pygnulib import constants
  49 from pygnulib import classes
  50
  51
  52 #===============================================================================
  53 # Define global constants
  54 #===============================================================================
  55 APP = constants.APP
  56 DIRS = constants.DIRS
  57 ENCS = constants.ENCS
  58 UTILS = constants.UTILS
  59 MODES = constants.MODES
  60 TESTS = constants.TESTS
  61 joinpath = constants.joinpath
  62 copyfile = constants.copyfile
  63 isabs = os.path.isabs
  64 isdir = os.path.isdir
  65 isfile = os.path.isfile
  66
  67
  68 #===============================================================================
  69 # Define main part
  70 #===============================================================================
  71 def main():
  72     info = classes.GLInfo()
  73     parser = argparse.ArgumentParser(
  74         prog=constants.APP['name'],
  75         usage='gnulib-tool.py --help',
  76         add_help=False)
  77
  78     # Here we list the options in the order they are listed in the --help output.
  79
  80     # list
  81     parser.add_argument('--list',
  82                         dest='mode_list',
  83                         default=None,
  84                         action='store_true')
  85     # find
  86     parser.add_argument('--find',
  87                         dest='mode_find',
  88                         default=None,
  89                         action='store_true')
  90     # import
  91     parser.add_argument('--import',
  92                         dest='mode_import',
  93                         default=None,
  94                         action='store_true')
  95     # add-import
  96     parser.add_argument('--add-import',
  97                         dest='mode_add_import',
  98                         default=None,
  99                         action='store_true')
 100     # remove-import
 101     parser.add_argument('--remove-import',
 102                         dest='mode_remove_import',
 103                         default=None,
 104                         action='store_true')
 105     # update
 106     parser.add_argument('--update',
 107                         dest='mode_update',
 108                         default=None,
 109                         action='store_true')
 110     # create-testdir
 111     parser.add_argument('--create-testdir',
 112                         dest='mode_create_testdir',
 113                         default=None,
 114                         action='store_true')
 115     # create-megatestdir
 116     parser.add_argument('--create-megatestdir',
 117                         dest='mode_create_megatestdir',
 118                         default=None,
 119                         action='store_true')
 120     # test
 121     parser.add_argument('--test',
 122                         dest='mode_test',
 123                         default=None,
 124                         action='store_true')
 125     # megatest
 126     parser.add_argument('--megatest',
 127                         dest='mode_megatest',
 128                         default=None,
 129                         action='store_true')
 130     # extract-*
 131     parser.add_argument('--extract-description',
 132                         dest='mode_xdescription',
 133                         default=None,
 134                         action='store_true')
 135     parser.add_argument('--extract-comment',
 136                         dest='mode_xcomment',
 137                         default=None,
 138                         action='store_true')
 139     parser.add_argument('--extract-status',
 140                         dest='mode_xstatus',
 141                         default=None,
 142                         action='store_true')
 143     parser.add_argument('--extract-notice',
 144                         dest='mode_xnotice',
 145                         default=None,
 146                         action='store_true')
 147     parser.add_argument('--extract-applicability',
 148                         dest='mode_xapplicability',
 149                         default=None,
 150                         action='store_true')
 151     parser.add_argument('--extract-filelist',
 152                         dest='mode_xfilelist',
 153                         default=None,
 154                         action='store_true')
 155     parser.add_argument('--extract-dependencies',
 156                         dest='mode_xdependencies',
 157                         default=None,
 158                         action='store_true')
 159     parser.add_argument('--extract-autoconf-snippet',
 160                         dest='mode_xautoconf',
 161                         default=None,
 162                         action='store_true')
 163     parser.add_argument('--extract-automake-snippet',
 164                         dest='mode_xautomake',
 165                         default=None,
 166                         action='store_true')
 167     parser.add_argument('--extract-include-directive',
 168                         dest='mode_xinclude',
 169                         default=None,
 170                         action='store_true')
 171     parser.add_argument('--extract-link-directive',
 172                         dest='mode_xlink',
 173                         default=None,
 174                         action='store_true')
 175     parser.add_argument('--extract-license',
 176                         dest='mode_xlicense',
 177                         default=None,
 178                         action='store_true')
 179     parser.add_argument('--extract-maintainer',
 180                         dest='mode_xmaintainer',
 181                         default=None,
 182                         action='store_true')
 183     parser.add_argument('--extract-tests-module',
 184                         dest='mode_xtests',
 185                         default=None,
 186                         action='store_true')
 187     # copy-file
 188     parser.add_argument('--copy-file',
 189                         dest='mode_copy_file',
 190                         default=None,
 191                         action='store_true')
 192     # help
 193     parser.add_argument('--help', '--hel', '--he', '--h',
 194                         dest='help',
 195                         default=None,
 196                         action='store_true')
 197     # version
 198     parser.add_argument('--version', '--versio', '--versi', '--vers',
 199                         dest='version',
 200                         default=None,
 201                         action='store_true')
 202     # no-changelog: a no-op for backward compatibility
 203     parser.add_argument('--no-changelog',
 204                         dest='changelog',
 205                         default=None,
 206                         action='store_false')
 207     # destdir
 208     parser.add_argument('--dir',
 209                         dest='destdir',
 210                         default=None,
 211                         nargs=1)
 212     # localpath
 213     parser.add_argument('--local-dir',
 214                         action='append',
 215                         dest='localpath',
 216                         default=None,
 217                         nargs=1)
 218     # cache-modules: a no-op for backward compatibility
 219     parser.add_argument('--cache-modules',
 220                         dest='cache_modules',
 221                         default=None,
 222                         action='store_true')
 223     parser.add_argument('--no-cache-modules',
 224                         dest='cache_modules',
 225                         default=None,
 226                         action='store_false')
 227     # verbose
 228     parser.add_argument('--verbose',
 229                         default=0,
 230                         action='count')
 231     # quiet
 232     parser.add_argument('--quiet',
 233                         default=0,
 234                         action='count')
 235     # dryrun
 236     parser.add_argument('--dry-run',
 237                         dest='dryrun',
 238                         default=None,
 239                         action='store_true')
 240     # inctests
 241     parser.add_argument('--with-tests',
 242                         dest='inctests',
 243                         default=None,
 244                         action='store_true')
 245     parser.add_argument('--without-tests',
 246                         dest='inctests',
 247                         default=None,
 248                         action='store_false')
 249     # obsolete
 250     parser.add_argument('--with-obsolete',
 251                         dest='obsolete',
 252                         default=None,
 253                         action='store_true')
 254     # c++-tests
 255     parser.add_argument('--with-c++-tests',
 256                         dest='inc_cxx_tests',
 257                         default=None,
 258                         action='store_true')
 259     parser.add_argument('--without-c++-tests',
 260                         dest='excl_cxx_tests',
 261                         default=None,
 262                         action='store_true')
 263     # longrunning-tests
 264     parser.add_argument('--with-longrunning-tests',
 265                         dest='inc_longrunning_tests',
 266                         default=None,
 267                         action='store_true')
 268     parser.add_argument('--without-longrunning-tests',
 269                         dest='excl_longrunning_tests',
 270                         default=None,
 271                         action='store_true')
 272     # privileged-tests
 273     parser.add_argument('--with-privileged-tests',
 274                         dest='inc_privileged_tests',
 275                         default=None,
 276                         action='store_true')
 277     parser.add_argument('--without-privileged-tests',
 278                         dest='excl_privileged_tests',
 279                         default=None,
 280                         action='store_true')
 281     # unportable-tests
 282     parser.add_argument('--with-unportable-tests',
 283                         dest='inc_unportable_tests',
 284                         default=None,
 285                         action='store_true')
 286     parser.add_argument('--without-unportable-tests',
 287                         dest='excl_unportable_tests',
 288                         default=None,
 289                         action='store_true')
 290     # all-tests
 291     parser.add_argument('--with-all-tests',
 292                         dest='alltests',
 293                         default=None,
 294                         action='store_true')
 295     # avoids
 296     parser.add_argument('--avoid',
 297                         dest='avoids',
 298                         default=None,
 299                         action='append',
 300                         nargs=1)
 301     # conditional-dependencies
 302     parser.add_argument('--conditional-dependencies',
 303                         dest='cond_dependencies',
 304                         default=None,
 305                         action="store_true")
 306     parser.add_argument('--no-conditional-dependencies',
 307                         dest='cond_dependencies',
 308                         default=None,
 309                         action="store_false")
 310     # libtool
 311     parser.add_argument('--libtool',
 312                         dest='libtool',
 313                         default=None,
 314                         action="store_true")
 315     parser.add_argument('--no-libtool',
 316                         dest='libtool',
 317                         default=None,
 318                         action="store_false")
 319     # libname
 320     parser.add_argument('--lib',
 321                         dest='libname',
 322                         default=None,
 323                         nargs=1)
 324     # sourcebase
 325     parser.add_argument('--source-base',
 326                         dest='sourcebase',
 327                         default=None,
 328                         nargs=1)
 329     # m4base
 330     parser.add_argument('--m4-base',
 331                         dest='m4base',
 332                         default=None,
 333                         nargs=1)
 334     # pobase
 335     parser.add_argument('--po-base',
 336                         dest='pobase',
 337                         default=None,
 338                         nargs=1)
 339     # docbase
 340     parser.add_argument('--doc-base',
 341                         dest='docbase',
 342                         default=None,
 343                         nargs=1)
 344     # testsbase
 345     parser.add_argument('--tests-base',
 346                         dest='testsbase',
 347                         default=None,
 348                         nargs=1)
 349     # auxdir
 350     parser.add_argument('--aux-dir',
 351                         dest='auxdir',
 352                         default=None,
 353                         nargs=1)
 354     # lgpl
 355     parser.add_argument('--lgpl',
 356                         dest='lgpl',
 357                         default=None,
 358                         action='append',
 359                         choices=['2', '3orGPLv2', '3'],
 360                         nargs='?')
 361     # makefile-name
 362     parser.add_argument('--makefile-name',
 363                         dest='makefile_name',
 364                         default=None,
 365                         nargs=1)
 366     # macro-prefix
 367     parser.add_argument('--macro-prefix',
 368                         dest='macro_prefix',
 369                         default=None,
 370                         nargs=1)
 371     # po-domain
 372     parser.add_argument('--po-domain',
 373                         dest='podomain',
 374                         default=None,
 375                         nargs=1)
 376     # witness-c-macro
 377     parser.add_argument('--witness-c-macro',
 378                         dest='witness_c_macro',
 379                         default=None,
 380                         nargs=1)
 381     # vc-files
 382     parser.add_argument('--vc-files',
 383                         dest='vc_files',
 384                         default=None,
 385                         action='store_true')
 386     parser.add_argument('--no-vc-files',
 387                         dest='vc_files',
 388                         default=None,
 389                         action='store_false')
 390     # single-configure
 391     parser.add_argument('--single-configure',
 392                         dest='single_configure',
 393                         default=None,
 394                         action='store_true')
 395     # symlink
 396     parser.add_argument('-s', '--symbolic', '--symlink',
 397                         dest='symlink',
 398                         default=None,
 399                         action='store_true')
 400     # local-symlink
 401     parser.add_argument('--local-symlink',
 402                         dest='lsymlink',
 403                         default=None,
 404                         action='store_true')
 405     # All other arguments are collected.
 406     parser.add_argument("non_option_arguments",
 407                         nargs='*')
 408
 409     # Parse the given arguments. Don't signal an error if non-option arguments
 410     # occur between or after options.
 411     cmdargs, unhandled = parser.parse_known_args()
 412
 413     # Handle --help and --version, ignoring all other options.
 414     if cmdargs.help != None:
 415         print(info.usage())
 416         sys.exit(0)
 417     if cmdargs.version != None:
 418         version = info.version()
 419         if version != '':
 420             version = ' ' + version
 421         message = '''gnulib-tool (%s %s)%s\n%s\n%s\n\nWritten by %s.''' \
 422             % (info.package(), info.date(), version, info.copyright(),
 423                info.license(), info.authors())
 424         print(message)
 425         sys.exit(0)
 426
 427     # Report unhandled arguments.
 428     for arg in unhandled:
 429         if arg.startswith('-'):
 430             message = '%s: Unrecognized option \'%s\'.\n' % (constants.APP['name'], arg)
 431             message += 'Try \'gnulib-tool --help\' for more information.\n'
 432             sys.stderr.write(message)
 433             sys.exit(1)
 434     # By now, all unhandled arguments were non-options.
 435     cmdargs.non_option_arguments += unhandled
 436
 437     # Determine when user tries to combine modes.
 438     args = [
 439         cmdargs.mode_list,
 440         cmdargs.mode_find,
 441         cmdargs.mode_import,
 442         cmdargs.mode_add_import,
 443         cmdargs.mode_remove_import,
 444         cmdargs.mode_update,
 445         cmdargs.mode_create_testdir,
 446         cmdargs.mode_create_megatestdir,
 447         cmdargs.mode_test,
 448         cmdargs.mode_megatest,
 449         cmdargs.mode_xdescription,
 450         cmdargs.mode_xcomment,
 451         cmdargs.mode_xstatus,
 452         cmdargs.mode_xnotice,
 453         cmdargs.mode_xapplicability,
 454         cmdargs.mode_xfilelist,
 455         cmdargs.mode_xdependencies,
 456         cmdargs.mode_xautoconf,
 457         cmdargs.mode_xautomake,
 458         cmdargs.mode_xinclude,
 459         cmdargs.mode_xlink,
 460         cmdargs.mode_xlicense,
 461         cmdargs.mode_xmaintainer,
 462         cmdargs.mode_xtests,
 463         cmdargs.mode_copy_file,
 464     ]
 465     overflow = [ arg
 466                  for arg in args
 467                  if arg != None ]
 468     if len(overflow) > 1:
 469         message = '%s: Unable to combine different modes of work.\n' % constants.APP['name']
 470         message += 'Try \'gnulib-tool --help\' for more information.\n'
 471         sys.stderr.write(message)
 472         sys.exit(1)
 473
 474     # Determine selected mode.
 475     mode = None
 476     modules = None
 477     files = None
 478     if cmdargs.mode_list != None:
 479         mode = 'list'
 480     if cmdargs.mode_find != None:
 481         mode = 'find'
 482         files = list(cmdargs.non_option_arguments)
 483     if cmdargs.mode_import != None:
 484         mode = 'import'
 485         modules = list(cmdargs.non_option_arguments)
 486     if cmdargs.mode_add_import != None:
 487         mode = 'add-import'
 488         modules = list(cmdargs.non_option_arguments)
 489     if cmdargs.mode_remove_import != None:
 490         mode = 'remove-import'
 491         modules = list(cmdargs.non_option_arguments)
 492     if cmdargs.mode_update != None:
 493         mode = 'update'
 494         if len(cmdargs.non_option_arguments) > 0:
 495             message = '%s: too many arguments in \'update\' mode\n'
 496             message += 'Try \'gnulib-tool --help\' for more information.\n'
 497             message += 'If you really want to modify the gnulib configuration of your project,\n'
 498             message += 'you need to use \'gnulib-tool --import\' - at your own risk!\n'
 499             sys.stderr.write(message)
 500             sys.exit(1)
 501     if cmdargs.mode_create_testdir != None:
 502         mode = 'create-testdir'
 503         modules = list(cmdargs.non_option_arguments)
 504     if cmdargs.mode_create_megatestdir != None:
 505         mode = 'create-megatestdir'
 506         modules = list(cmdargs.non_option_arguments)
 507     if cmdargs.mode_test != None:
 508         mode = 'test'
 509         modules = list(cmdargs.non_option_arguments)
 510     if cmdargs.mode_megatest != None:
 511         mode = 'megatest'
 512         modules = list(cmdargs.non_option_arguments)
 513     if cmdargs.mode_xdescription != None:
 514         mode = 'extract-description'
 515         modules = list(cmdargs.non_option_arguments)
 516     if cmdargs.mode_xcomment != None:
 517         mode = 'extract-comment'
 518         modules = list(cmdargs.non_option_arguments)
 519     if cmdargs.mode_xstatus != None:
 520         mode = 'extract-status'
 521         modules = list(cmdargs.non_option_arguments)
 522     if cmdargs.mode_xnotice != None:
 523         mode = 'extract-notice'
 524         modules = list(cmdargs.non_option_arguments)
 525     if cmdargs.mode_xapplicability != None:
 526         mode = 'extract-applicability'
 527         modules = list(cmdargs.non_option_arguments)
 528     if cmdargs.mode_xfilelist != None:
 529         mode = 'extract-filelist'
 530         modules = list(cmdargs.non_option_arguments)
 531     if cmdargs.mode_xautoconf != None:
 532         mode = 'extract-autoconf-snippet'
 533         modules = list(cmdargs.non_option_arguments)
 534     if cmdargs.mode_xautomake != None:
 535         mode = 'extract-automake-snippet'
 536         modules = list(cmdargs.non_option_arguments)
 537     if cmdargs.mode_xdependencies != None:
 538         mode = 'extract-dependencies'
 539         modules = list(cmdargs.non_option_arguments)
 540     if cmdargs.mode_xinclude != None:
 541         mode = 'extract-include-directive'
 542         modules = list(cmdargs.non_option_arguments)
 543     if cmdargs.mode_xlink != None:
 544         mode = 'extract-link-directive'
 545         modules = list(cmdargs.non_option_arguments)
 546     if cmdargs.mode_xlicense != None:
 547         mode = 'extract-license'
 548         modules = list(cmdargs.non_option_arguments)
 549     if cmdargs.mode_xmaintainer != None:
 550         mode = 'extract-maintainer'
 551         modules = list(cmdargs.non_option_arguments)
 552     if cmdargs.mode_xtests != None:
 553         mode = 'extract-tests-module'
 554         modules = list(cmdargs.non_option_arguments)
 555     if cmdargs.mode_copy_file != None:
 556         mode = 'copy-file'
 557         if len(cmdargs.non_option_arguments) < 1 or len(cmdargs.non_option_arguments) > 2:
 558             message = '%s: *** ' % constants.APP['name']
 559             message += 'invalid number of arguments for --%s\n' % mode
 560             message += 'Try \'gnulib-tool --help\' for more information.\n'
 561             message += '%s: *** Stop.\n' % constants.APP['name']
 562             sys.stderr.write(message)
 563             sys.exit(1)
 564         files = list(cmdargs.non_option_arguments)
 565
 566     if ((mode in ['import', 'add-import', 'remove-import']
 567          and (cmdargs.excl_cxx_tests or cmdargs.excl_longrunning_tests
 568               or cmdargs.excl_privileged_tests or cmdargs.excl_unportable_tests
 569               or cmdargs.single_configure))
 570         or (mode == 'update'
 571             and (cmdargs.localpath != None or cmdargs.libname != None
 572                  or cmdargs.sourcebase != None or cmdargs.m4base != None
 573                  or cmdargs.pobase != None or cmdargs.docbase != None
 574                  or cmdargs.testsbase != None or cmdargs.auxdir != None
 575                  or cmdargs.inctests != None or cmdargs.obsolete != None
 576                  or cmdargs.inc_cxx_tests != None
 577                  or cmdargs.inc_longrunning_tests != None
 578                  or cmdargs.inc_privileged_tests != None
 579                  or cmdargs.inc_unportable_tests != None
 580                  or cmdargs.alltests != None
 581                  or cmdargs.excl_cxx_tests != None
 582                  or cmdargs.excl_longrunning_tests != None
 583                  or cmdargs.excl_privileged_tests != None
 584                  or cmdargs.excl_unportable_tests != None
 585                  or cmdargs.avoids != None or cmdargs.lgpl != None
 586                  or cmdargs.makefile_name != None
 587                  or cmdargs.macro_prefix != None or cmdargs.podomain != None
 588                  or cmdargs.witness_c_macro != None or cmdargs.vc_files != None))):
 589         message = '%s: *** ' % constants.APP['name']
 590         message += 'invalid options for --%s mode\n' % mode
 591         message += 'Try \'gnulib-tool --help\' for more information.\n'
 592         message += '%s: *** Stop.\n' % constants.APP['name']
 593         sys.stderr.write(message)
 594         sys.exit(1)
 595     if cmdargs.pobase != None and cmdargs.podomain == None:
 596         message = '%s: *** ' % constants.APP['name']
 597         message += 'together with --po-base, you need to specify --po-domain\n'
 598         message += 'Try \'gnulib-tool --help\' for more information.\n'
 599         message += '%s: *** Stop.\n' % constants.APP['name']
 600         sys.stderr.write(message)
 601         sys.exit(1)
 602     if cmdargs.pobase == None and cmdargs.podomain != None:
 603         message = '%s: warning: --po-domain has no effect without a --po-base option\n' % constants.APP['name']
 604         sys.stderr.write(message)
 605
 606     # Determine specific settings.
 607     destdir = cmdargs.destdir
 608     if destdir != None:
 609         destdir = cmdargs.destdir[0]
 610     localpath = cmdargs.localpath
 611     if localpath != None:
 612         localpath = [ dir
 613                       for list1 in localpath
 614                       for dir in list1 ]
 615     libname = cmdargs.libname
 616     if libname != None:
 617         libname = cmdargs.libname[0]
 618     auxdir = cmdargs.auxdir
 619     if auxdir != None:
 620         auxdir = cmdargs.auxdir[0]
 621     sourcebase = cmdargs.sourcebase
 622     if sourcebase != None:
 623         sourcebase = cmdargs.sourcebase[0]
 624     m4base = cmdargs.m4base
 625     if m4base != None:
 626         m4base = cmdargs.m4base[0]
 627     pobase = cmdargs.pobase
 628     if pobase != None:
 629         pobase = cmdargs.pobase[0]
 630     testsbase = cmdargs.testsbase
 631     if testsbase != None:
 632         testsbase = cmdargs.testsbase[0]
 633     dryrun = cmdargs.dryrun
 634     verbose = -cmdargs.quiet + cmdargs.verbose
 635     inctests = cmdargs.inctests
 636     # Canonicalize the inctests variable.
 637     if inctests == None:
 638         if mode in ['import', 'add-import', 'remove-import', 'update']:
 639             inctests = False
 640         elif mode in ['create-testdir', 'create-megatestdir', 'test', 'megatest']:
 641             inctests = True
 642     incl_test_categories = []
 643     if inctests:
 644         incl_test_categories += [constants.TESTS['tests']]
 645     if cmdargs.obsolete:
 646         incl_test_categories += [constants.TESTS['obsolete']]
 647     if cmdargs.inc_cxx_tests:
 648         incl_test_categories += [constants.TESTS['cxx-tests']]
 649     if cmdargs.inc_longrunning_tests:
 650         incl_test_categories += [constants.TESTS['longrunning-tests']]
 651     if cmdargs.inc_privileged_tests:
 652         incl_test_categories += [constants.TESTS['privileged-tests']]
 653     if cmdargs.inc_unportable_tests:
 654         incl_test_categories += [constants.TESTS['unportable-tests']]
 655     if cmdargs.alltests:
 656         incl_test_categories += [constants.TESTS['all-tests']]
 657     excl_test_categories = []
 658     if cmdargs.excl_cxx_tests:
 659         excl_test_categories += [constants.TESTS['cxx-tests']]
 660     if cmdargs.excl_longrunning_tests:
 661         excl_test_categories += [constants.TESTS['longrunning-tests']]
 662     if cmdargs.excl_privileged_tests:
 663         excl_test_categories += [constants.TESTS['privileged-tests']]
 664     if cmdargs.excl_unportable_tests:
 665         excl_test_categories += [constants.TESTS['unportable-tests']]
 666     lgpl = cmdargs.lgpl
 667     if lgpl != None:
 668         lgpl = lgpl[-1]
 669         if lgpl == None:
 670             lgpl = True
 671     cond_dependencies = cmdargs.cond_dependencies
 672     libtool = cmdargs.libtool
 673     makefile_name = cmdargs.makefile_name
 674     if makefile_name != None:
 675         makefile_name = makefile_name[0]
 676     macro_prefix = cmdargs.macro_prefix
 677     if macro_prefix != None:
 678         macro_prefix = macro_prefix[0]
 679     podomain = cmdargs.podomain
 680     if podomain != None:
 681         podomain = podomain[0]
 682     witness_c_macro = cmdargs.witness_c_macro
 683     if witness_c_macro != None:
 684         witness_c_macro = witness_c_macro[0]
 685     vc_files = cmdargs.vc_files
 686     avoids = cmdargs.avoids
 687     if avoids != None:
 688         avoids = [ module
 689                    for list1 in avoids
 690                    for module in list1 ]
 691     symlink = cmdargs.symlink == True
 692     lsymlink = cmdargs.lsymlink == True
 693     single_configure = cmdargs.single_configure
 694     docbase = None
 695
 696     # Create pygnulib configuration.
 697     config = classes.GLConfig(
 698         destdir=destdir,
 699         localpath=localpath,
 700         m4base=m4base,
 701         auxdir=auxdir,
 702         modules=modules,
 703         avoids=avoids,
 704         sourcebase=sourcebase,
 705         pobase=pobase,
 706         docbase=docbase,
 707         testsbase=testsbase,
 708         incl_test_categories=incl_test_categories,
 709         excl_test_categories=excl_test_categories,
 710         libname=libname,
 711         lgpl=lgpl,
 712         makefile_name=makefile_name,
 713         libtool=libtool,
 714         conddeps=cond_dependencies,
 715         macro_prefix=macro_prefix,
 716         podomain=podomain,
 717         witness_c_macro=witness_c_macro,
 718         vc_files=vc_files,
 719         symbolic=symlink,
 720         lsymbolic=lsymlink,
 721         single_configure=single_configure,
 722         verbose=verbose,
 723         dryrun=dryrun,
 724     )
 725
 726     # Work in the given mode.
 727     if mode == 'list':
 728         modulesystem = classes.GLModuleSystem(config)
 729         listing = modulesystem.list()
 730         result = '\n'.join(listing)
 731         os.rmdir(config['tempdir'])
 732         print(result)
 733
 734     elif mode == 'find':
 735         modulesystem = classes.GLModuleSystem(config)
 736         for filename in files:
 737             if (isfile(joinpath(DIRS['root'], filename))
 738                     or (localpath != None
 739                         and any([ isfile(joinpath(localdir, filename))
 740                                   for localdir in localpath ]))):
 741                 # Convert the file name to a POSIX basic regex.
 742                 # Needs to handle . [ \ * ^ $.
 743                 filename_regex = filename.replace('\\', '\\\\').replace('[', '\\[').replace('^', '\\^')
 744                 filename_regex = re.compile('([.*$])').sub('[\\1]', filename_regex)
 745                 filename_line_regex = '^' + filename_regex + '$'
 746                 # Read module candidates from gnulib root directory.
 747                 command = "find modules -type f -print | xargs -n 100 grep -l %s /dev/null | sed -e 's,^modules/,,'" % shlex.quote(filename_line_regex)
 748                 os.chdir(constants.DIRS['root'])
 749                 with sp.Popen(command, shell=True, stdout=sp.PIPE) as proc:
 750                     result = proc.stdout.read().decode("UTF-8")
 751                 os.chdir(DIRS['cwd'])
 752                 # Read module candidates from local directories.
 753                 if localpath != None and len(localpath) > 0:
 754                     command = "find modules -type f -print | xargs -n 100 grep -l %s /dev/null | sed -e 's,^modules/,,' -e 's,\\.diff$,,'" % shlex.quote(filename_line_regex)
 755                     for localdir in localpath:
 756                         os.chdir(localdir)
 757                         with sp.Popen(command, shell=True, stdout=sp.PIPE) as proc:
 758                             result += proc.stdout.read().decode("UTF-8")
 759                         os.chdir(DIRS['cwd'])
 760                 listing = [ line
 761                             for line in result.split('\n')
 762                             if line.strip() ]
 763                 # Remove modules/ prefix from each file name.
 764                 pattern = re.compile('^modules/')
 765                 listing = [ pattern.sub('', line)
 766                             for line in listing ]
 767                 # Filter out undesired file names.
 768                 listing = [ line
 769                             for line in listing
 770                             if modulesystem.file_is_module(line) ]
 771                 candidates = sorted(set(listing))
 772                 for name in candidates:
 773                     module = modulesystem.find(name)
 774                     if module:  # Ignore module candidates that don't actually exist.
 775                         if module.getFiles():
 776                             print(name)
 777             else:
 778                 message = '%s: warning: file %s does not exist\n' % (constants.APP['name'], filename)
 779                 sys.stderr.write(message)
 780
 781     elif mode in ['import', 'add-import', 'remove-import', 'update']:
 782         mode = MODES[mode]
 783         if not destdir:
 784             destdir = '.'
 785         config.setDestDir(destdir)
 786
 787         if mode == MODES['import']:
 788             # Set variables to default values.
 789             if not sourcebase:
 790                 sourcebase = 'lib'
 791             if not m4base:
 792                 m4base = 'm4'
 793             if not docbase:
 794                 docbase = 'doc'
 795             if not testsbase:
 796                 testsbase = 'tests'
 797             if not macro_prefix:
 798                 macro_prefix = 'gl'
 799             config.setSourceBase(sourcebase)
 800             config.setM4Base(m4base)
 801             config.setDocBase(docbase)
 802             config.setTestsBase(testsbase)
 803             config.setMacroPrefix(macro_prefix)
 804
 805             # Perform GLImport actions.
 806             importer = classes.GLImport(config, mode)
 807             filetable, transformers = importer.prepare()
 808             importer.execute(filetable, transformers)
 809
 810         else:  # if mode != MODE['--import']
 811             if m4base:
 812                 if not isfile(joinpath(destdir, m4base, 'gnulib-cache.m4')):
 813                     if not sourcebase:
 814                         sourcebase = 'lib'
 815                     if not docbase:
 816                         docbase = 'doc'
 817                     if not testsbase:
 818                         testsbase = 'tests'
 819                     if not macro_prefix:
 820                         macro_prefix = 'gl'
 821                     config.setSourceBase(sourcebase)
 822                     config.setM4Base(m4base)
 823                     config.setDocBase(docbase)
 824                     config.setTestsBase(testsbase)
 825                     config.setMacroPrefix(macro_prefix)
 826                 # Perform GLImport actions.
 827                 importer = classes.GLImport(config, mode)
 828                 filetable, transformers = importer.prepare()
 829                 importer.execute(filetable, transformers)
 830             else:  # if not m4base
 831                 m4dirs = list()
 832                 dirisnext = bool()
 833                 filepath = joinpath(destdir, 'Makefile.am')
 834                 if isfile(filepath):
 835                     with codecs.open(filepath, 'rb', 'UTF-8') as file:
 836                         data = file.read()
 837                         data = data.split('ACLOCAL_AMFLAGS')[1]
 838                         data = data[data.find('=') + 1:data.find('\n')]
 839                     aclocal_amflags = data.split()
 840                     for aclocal_amflag in aclocal_amflags:
 841                         if dirisnext:
 842                             if not isabs(aclocal_amflag):
 843                                 m4dirs += [aclocal_amflag]
 844                         else:  # if not dirisnext
 845                             if aclocal_amflag == '-I':
 846                                 dirisnext = True
 847                             else:  # if aclocal_amflag != '-I'
 848                                 dirisnext = False
 849                 else:  # if not isfile(filepath)
 850                     filepath = joinpath(destdir, 'aclocal.m4')
 851                     if isfile(filepath):
 852                         pattern = re.compile(r'm4_include\(\[(.*?)\]\)')
 853                         with codecs.open(filepath, 'rb', 'UTF-8') as file:
 854                             m4dirs = pattern.findall(file.read())
 855                         m4dirs = [ os.path.dirname(m4dir)
 856                                    for m4dir in m4dirs ]
 857                         m4dirs = [ m4dir
 858                                    for m4dir in m4dirs
 859                                    if isfile(joinpath(destdir, m4dir, 'gnulib-cache.m4')) ]
 860                         m4dirs = sorted(set(m4dirs))
 861                 if len(m4dirs) == 0:
 862                     # First use of gnulib in a package.
 863                     # Any number of additional modules can be given.
 864                     if not sourcebase:
 865                         sourcebase = 'lib'
 866                     m4base = 'm4'
 867                     if not docbase:
 868                         docbase = 'doc'
 869                     if not testsbase:
 870                         testsbase = 'tests'
 871                     if not macro_prefix:
 872                         macro_prefix = 'gl'
 873                     config.setSourceBase(sourcebase)
 874                     config.setM4Base(m4base)
 875                     config.setDocBase(docbase)
 876                     config.setTestsBase(testsbase)
 877                     config.setMacroPrefix(macro_prefix)
 878                     # Perform GLImport actions.
 879                     importer = classes.GLImport(config, mode)
 880                     filetable, transformers = importer.prepare()
 881                     importer.execute(filetable, transformers)
 882                 elif len(m4dirs) == 1:
 883                     m4base = m4dirs[-1]
 884                     config.setM4Base(m4base)
 885                     # Perform GLImport actions.
 886                     importer = classes.GLImport(config, mode)
 887                     filetable, transformers = importer.prepare()
 888                     importer.execute(filetable, transformers)
 889                 else:  # if len(m4dirs) > 1
 890                     for m4base in m4dirs:
 891                         config.setM4Base(m4base)
 892                         # Perform GLImport actions.
 893                         importer = classes.GLImport(config, mode)
 894                         filetable, transformers = importer.prepare()
 895                         importer.execute(filetable, transformers)
 896
 897     elif mode == 'create-testdir':
 898         if not destdir:
 899             message = '%s: *** ' % constants.APP['name']
 900             message += 'please specify --dir option\n'
 901             message += '%s: *** Stop.\n' % constants.APP['name']
 902             sys.stderr.write(message)
 903             sys.exit(1)
 904         if not auxdir:
 905             auxdir = 'build-aux'
 906         config.setAuxDir(auxdir)
 907         testdir = classes.GLTestDir(config, destdir)
 908         testdir.execute()
 909
 910     elif mode == 'create-megatestdir':
 911         if not destdir:
 912             message = '%s: *** ' % constants.APP['name']
 913             message += 'please specify --dir option\n'
 914             message += '%s: *** Stop.\n' % constants.APP['name']
 915             sys.stderr.write(message)
 916             sys.exit(1)
 917         if not auxdir:
 918             auxdir = 'build-aux'
 919         config.setAuxDir(auxdir)
 920         testdir = classes.GLMegaTestDir(config, destdir)
 921         testdir.execute()
 922
 923     elif mode == 'test':
 924         if not destdir:
 925             destdir = 'testdir%04d' % random.randrange(0, 9999)
 926         if not auxdir:
 927             auxdir = 'build-aux'
 928         config.setAuxDir(auxdir)
 929         testdir = classes.GLTestDir(config, destdir)
 930         testdir.execute()
 931         os.chdir(destdir)
 932         os.mkdir('build')
 933         os.chdir('build')
 934         try:  # Try to execute commands
 935             sp.call(['../configure'])
 936             sp.call([UTILS['make']])
 937             sp.call([UTILS['make'], 'check'])
 938             sp.call([UTILS['make'], 'distclean'])
 939         except Exception as error:
 940             sys.exit(1)
 941         args = ['find', '.', '-type', 'f', '-print']
 942         remaining = sp.check_output(args).decode(ENCS['shell'])
 943         lines = [ line.strip()
 944                   for line in remaining.split('\n')
 945                   if line.strip() ]
 946         remaining = ' '.join(lines)
 947         if remaining:
 948             message = 'Remaining files: %s\n' % remaining
 949             message += 'gnulib-tool: *** Stop.\n'
 950             sys.stderr.write(message)
 951             sys.exit(1)
 952         os.chdir('../..')
 953         sp.call(['rm', '-rf', destdir], shell=False)
 954
 955     elif mode == 'megatest':
 956         if not destdir:
 957             destdir = 'testdir %04d' % random.randrange(0, 9999)
 958         if not auxdir:
 959             auxdir = 'build-aux'
 960         config.setAuxDir(auxdir)
 961         testdir = classes.GLMegaTestDir(config, destdir)
 962         testdir.execute()
 963         os.chdir(destdir)
 964         os.mkdir('build')
 965         os.chdir('build')
 966         sp.call(['../configure'])
 967         sp.call([UTILS['make']])
 968         sp.call([UTILS['make'], 'check'])
 969         sp.call([UTILS['make'], 'distclean'])
 970         args = ['find', '.', '-type', 'f', '-print']
 971         remaining = sp.check_output(args).decode(ENCS['shell'])
 972         lines = [ line.strip()
 973                   for line in remaining.split('\n')
 974                   if line.strip() ]
 975         remaining = ' '.join(lines)
 976         if remaining:
 977             message = 'Remaining files: %s\n' % remaining
 978             message += 'gnulib-tool: *** Stop.\n'
 979             sys.stderr.write(message)
 980             sys.exit(1)
 981         os.chdir('../..')
 982         sp.call(['rm', '-rf', destdir], shell=False)
 983
 984     elif mode == 'extract-description':
 985         modulesystem = classes.GLModuleSystem(config)
 986         for name in modules:
 987             module = modulesystem.find(name)
 988             if module:
 989                 sys.stdout.write(module.getDescription())
 990
 991     elif mode == 'extract-comment':
 992         modulesystem = classes.GLModuleSystem(config)
 993         for name in modules:
 994             module = modulesystem.find(name)
 995             if module:
 996                 sys.stdout.write(module.getComment())
 997
 998     elif mode == 'extract-status':
 999         modulesystem = classes.GLModuleSystem(config)
1000         for name in modules:
1001             module = modulesystem.find(name)
1002             if module:
1003                 sys.stdout.write(module.getStatus())
1004
1005     elif mode == 'extract-notice':
1006         modulesystem = classes.GLModuleSystem(config)
1007         for name in modules:
1008             module = modulesystem.find(name)
1009             if module:
1010                 sys.stdout.write(module.getNotice())
1011
1012     elif mode == 'extract-applicability':
1013         modulesystem = classes.GLModuleSystem(config)
1014         for name in modules:
1015             module = modulesystem.find(name)
1016             if module:
1017                 print(module.getApplicability())
1018
1019     elif mode == 'extract-filelist':
1020         modulesystem = classes.GLModuleSystem(config)
1021         for name in modules:
1022             module = modulesystem.find(name)
1023             if module:
1024                 files = module.getFiles()
1025                 print('\n'.join(files))
1026
1027     elif mode == 'extract-dependencies':
1028         if avoids:
1029             message = '%s: *** ' % constants.APP['name']
1030             message += 'cannot combine --avoid and --extract-dependencies\n'
1031             message += '%s: *** Stop.\n' % constants.APP['name']
1032             sys.stderr.write(message)
1033             sys.exit(1)
1034         modulesystem = classes.GLModuleSystem(config)
1035         for name in modules:
1036             module = modulesystem.find(name)
1037             if module:
1038                 sys.stdout.write(module.getDependencies())
1039
1040     elif mode == 'extract-autoconf-snippet':
1041         modulesystem = classes.GLModuleSystem(config)
1042         for name in modules:
1043             module = modulesystem.find(name)
1044             if module:
1045                 sys.stdout.write(module.getAutoconfSnippet())
1046
1047     elif mode == 'extract-automake-snippet':
1048         modulesystem = classes.GLModuleSystem(config)
1049         for name in modules:
1050             module = modulesystem.find(name)
1051             if module:
1052                 sys.stdout.write(module.getAutomakeSnippet())
1053
1054     elif mode == 'extract-include-directive':
1055         modulesystem = classes.GLModuleSystem(config)
1056         for name in modules:
1057             module = modulesystem.find(name)
1058             if module:
1059                 sys.stdout.write(module.getInclude())
1060
1061     elif mode == 'extract-link-directive':
1062         modulesystem = classes.GLModuleSystem(config)
1063         for name in modules:
1064             module = modulesystem.find(name)
1065             if module:
1066                 sys.stdout.write(module.getLink())
1067
1068     elif mode == 'extract-license':
1069         modulesystem = classes.GLModuleSystem(config)
1070         for name in modules:
1071             module = modulesystem.find(name)
1072             if module:
1073                 print(module.getLicense())
1074
1075     elif mode == 'extract-maintainer':
1076         modulesystem = classes.GLModuleSystem(config)
1077         for name in modules:
1078             module = modulesystem.find(name)
1079             if module:
1080                 sys.stdout.write(module.getMaintainer())
1081
1082     elif mode == 'extract-tests-module':
1083         modulesystem = classes.GLModuleSystem(config)
1084         for name in modules:
1085             module = modulesystem.find(name)
1086             if module:
1087                 if module.getTestsModule():
1088                     print(module.getTestsName())
1089
1090     elif mode == 'copy-file':
1091         srcpath = files[0]
1092         # The second argument is the destination; either a directory ot a file.
1093         # It defaults to the current directory.
1094         if len(files) == 2:
1095             dest = files[1]
1096         else:  # if len(files) < 2
1097             dest = '.'
1098         if not auxdir:
1099             auxdir = 'build-aux'
1100         if not sourcebase:
1101             sourcebase = 'lib'
1102         if not m4base:
1103             m4base = 'm4'
1104         if not docbase:
1105             docbase = 'doc'
1106         if not testsbase:
1107             testsbase = 'tests'
1108         config.setAuxDir(auxdir)
1109         config.setSourceBase(sourcebase)
1110         config.setM4Base(m4base)
1111         config.setDocBase(docbase)
1112         config.setTestsBase(testsbase)
1113         filesystem = classes.GLFileSystem(config)
1114         lookedup, flag = filesystem.lookup(srcpath)
1115         if isdir(dest):
1116             destdir = dest
1117             if srcpath.startswith('build-aux/'):
1118                 destpath = constants.substart('build-aux/', '%s/' % auxdir, srcpath)
1119             elif srcpath.startswith('doc/'):
1120                 destpath = constants.substart('doc/', '%s/' % docbase, srcpath)
1121             elif srcpath.startswith('lib/'):
1122                 destpath = constants.substart('lib/', '%s/' % sourcebase, srcpath)
1123             elif srcpath.startswith('m4/'):
1124                 destpath = constants.substart('m4/', '%s/' % m4base, srcpath)
1125             elif srcpath.startswith('tests/'):
1126                 destpath = constants.substart('tests/', '%s/' % testsbase, srcpath)
1127             elif srcpath.startswith('top/'):
1128                 destpath = constants.substart('top/', '', srcpath)
1129             else:  # either case
1130                 destpath = srcpath
1131         else:  # if not isdir(dest)
1132             destdir = os.path.dirname(dest)
1133             destpath = os.path.basename(dest)
1134         # Create the directory for destfile.
1135         dirname = os.path.dirname(joinpath(destdir, destpath))
1136         if not config['dryrun']:
1137             if dirname and not isdir(dirname):
1138                 try:  # Try to create directories
1139                     os.makedirs(dirname)
1140                 except FileExistsError:
1141                     pass
1142         # Copy the file.
1143         assistant = classes.GLFileAssistant(config)
1144         tmpfile = assistant.tmpfilename(destpath)
1145         copyfile(lookedup, tmpfile)
1146         assistant.setOriginal(srcpath)
1147         assistant.config.setDestDir(destdir)
1148         assistant.setRewritten(destpath)
1149         if isfile(joinpath(destdir, destpath)):
1150             # The file already exists.
1151             assistant.update(lookedup, flag, tmpfile, True)
1152         else:  # if not isfile(joinpath(destdir, destpath))
1153             # Install the file.
1154             # Don't protest if the file should be there but isn't: it happens
1155             # frequently that developers don't put autogenerated files under
1156             # version control.
1157             assistant.add(lookedup, flag, tmpfile)
1158         if isfile(tmpfile):
1159             os.remove(tmpfile)
1160
1161     else:
1162         message = '%s: *** ' % constants.APP['name']
1163         message += 'no mode specified\n'
1164         message += '%s: *** Stop.\n' % constants.APP['name']
1165         sys.stderr.write(message)
1166         sys.exit(1)
1167
1168
1169 if __name__ == '__main__':
1170     try:  # Try to execute
1171         main()
1172     except classes.GLError as error:
1173         errmode = 0  # gnulib-style errors
1174         errno = error.errno
1175         errinfo = error.errinfo
1176         if errmode == 0:
1177             message = '%s: *** ' % constants.APP['name']
1178             if errinfo == None:
1179                 errinfo = ''
1180             if errno == 1:
1181                 message += 'file %s not found' % errinfo
1182             elif errno == 2:
1183                 message += 'patch file %s didn\'t apply cleanly' % errinfo
1184             elif errno == 3:
1185                 message += 'cannot find %s - make sure you run gnulib-tool from within your package\'s directory' % errinfo
1186             elif errno == 4:
1187                 message += 'minimum supported autoconf version is 2.59. Try adding'
1188                 message += 'AC_PREREQ([%s])' % constants.DEFAULT_AUTOCONF_MINVERSION
1189                 message += ' to your configure.ac.'
1190             elif errno == 5:
1191                 message += '%s is expected to contain gl_M4_BASE([%s])' % (repr(os.path.join(errinfo, 'gnulib-comp.m4')), repr(errinfo))
1192             elif errno == 6:
1193                 message += 'missing --source-base option'
1194             elif errno == 7:
1195                 message += 'missing --doc-base option. --doc-base has been introduced '
1196                 message += 'on 2006-07-11; if your last invocation of \'gnulib-tool '
1197                 message += '--import\' is before that date, you need to run'
1198                 message += '\'gnulib-tool --import\' once, with a --doc-base option.'
1199             elif errno == 8:
1200                 message += 'missing --tests-base option'
1201             elif errno == 9:
1202                 message += 'missing --lib option'
1203             elif errno == 10:
1204                 message = 'gnulib-tool: option --conditional-dependencies is not supported with --with-tests'
1205             elif errno == 11:
1206                 incompatibilities = ''
1207                 message += 'incompatible license on modules:%s' % constants.NL
1208                 for pair in errinfo:
1209                     incompatibilities += pair[0]
1210                     incompatibilities += ' %s' % pair[1]
1211                     incompatibilities += constants.NL
1212                 tempname = mktemp()
1213                 with codecs.open(tempname, 'wb', 'UTF-8') as file:
1214                     file.write(incompatibilities)
1215                 sed_table = 's,^\\([^ ]*\\) ,\\1' + ' ' * 51 + ',\n'
1216                 sed_table += 's,^\\(' + '.' * 49 + '[^ ]*\\) *,' + ' ' * 17 + '\\1 ,'
1217                 args = ['sed', '-e', sed_table, tempname]
1218                 incompatibilities = sp.check_output(args).decode(ENCS['default'])
1219                 message += incompatibilities
1220                 os.remove(tempname)
1221             elif errno == 12:
1222                 message += 'refusing to do nothing'
1223             elif errno == 13:
1224                 message += 'could not create directory %s' % errinfo
1225             elif errno == 14:
1226                 message += 'could not delete file %s' % errinfo
1227             elif errno == 15:
1228                 message += 'could not create file %s' % errinfo
1229             elif errno == 16:
1230                 message += 'could not transform file %s' % errinfo
1231             elif errno == 17:
1232                 message += 'could not update file %s' % errinfo
1233             elif errno == 18:
1234                 message += 'module %s lacks a license' % errinfo
1235             elif errno == 19:
1236                 message += 'could not create destination directory: %s' % errinfo
1237             message += '\n%s: *** Stop.\n' % constants.APP['name']
1238             sys.stderr.write(message)
1239             sys.exit(1)