python/samba/tests/usage.py

   1 # Unix SMB/CIFS implementation.
   2 # Copyright © Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import subprocess
  19 from samba.tests import TestCase, check_help_consistency
  20 import re
  21 import stat
  22
  23 if 'SRCDIR_ABS' in os.environ:
  24     BASEDIR = os.environ['SRCDIR_ABS']
  25 else:
  26     BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
  27                                            '../../..'))
  28
  29 TEST_DIRS = [
  30     "bootstrap",
  31     "testdata",
  32     "ctdb",
  33     "dfs_server",
  34     "pidl",
  35     "auth",
  36     "packaging",
  37     "python",
  38     "include",
  39     "nsswitch",
  40     "libcli",
  41     "coverity",
  42     "release-scripts",
  43     "testprogs",
  44     "bin",
  45     "source3",
  46     "docs-xml",
  47     "buildtools",
  48     "file_server",
  49     "dynconfig",
  50     "source4",
  51     "tests",
  52     "libds",
  53     "selftest",
  54     "lib",
  55     "script",
  56     "traffic",
  57     "testsuite",
  58     "libgpo",
  59     "wintest",
  60     "librpc",
  61 ]
  62
  63
  64 EXCLUDE_USAGE = {
  65     'script/autobuild.py',  # defaults to mount /memdisk/
  66     'script/bisect-test.py',
  67     'ctdb/utils/etcd/ctdb_etcd_lock',
  68     'selftest/filter-subunit',
  69     'selftest/format-subunit',
  70     'bin/gen_output.py',  # too much output!
  71     'source4/scripting/bin/gen_output.py',
  72     'lib/ldb/tests/python/index.py',
  73     'lib/ldb/tests/python/api.py',
  74     'source4/selftest/tests.py',
  75     'buildtools/bin/waf',
  76     'selftest/tap2subunit',
  77     'script/show_test_time',
  78     'source4/scripting/bin/subunitrun',
  79     'bin/samba_downgrade_db',
  80     'source4/scripting/bin/samba_downgrade_db',
  81     'source3/selftest/tests.py',
  82     'selftest/tests.py',
  83     'python/samba/subunit/run.py',
  84     'bin/python/samba/subunit/run.py',
  85     'lib/compression/tests/scripts/three-byte-hash',
  86 }
  87
  88 EXCLUDE_HELP = {
  89     'selftest/tap2subunit',
  90     'wintest/test-s3.py',
  91     'wintest/test-s4-howto.py',
  92 }
  93
  94
  95 EXCLUDE_DIRS = {
  96     'source3/script/tests',
  97     'python/examples',
  98     'source4/dsdb/tests/python',
  99     'bin/ab',
 100     'bin/python/samba/tests',
 101     'bin/python/samba/tests/dcerpc',
 102     'bin/python/samba/tests/krb5',
 103     'bin/python/samba/tests/ndr',
 104     'python/samba/tests',
 105     'python/samba/tests/bin',
 106     'python/samba/tests/dcerpc',
 107     'python/samba/tests/krb5',
 108     'python/samba/tests/ndr',
 109 }
 110
 111
 112 def _init_git_file_finder():
 113     """Generate a function that quickly answers the question:
 114     'is this a git file?'
 115     """
 116     git_file_cache = set()
 117     p = subprocess.run(['git',
 118                         '-C', BASEDIR,
 119                         'ls-files',
 120                         '-z'],
 121                        stdout=subprocess.PIPE)
 122     if p.returncode == 0:
 123         for fn in p.stdout.split(b'\0'):
 124             git_file_cache.add(os.path.join(BASEDIR, fn.decode('utf-8')))
 125     return git_file_cache.__contains__
 126
 127
 128 is_git_file = _init_git_file_finder()
 129
 130
 131 def script_iterator(d=BASEDIR, cache=None,
 132                     shebang_filter=None,
 133                     filename_filter=None,
 134                     subdirs=None):
 135     if subdirs is None:
 136         subdirs = TEST_DIRS
 137     if not cache:
 138         safename = re.compile(r'\W+').sub
 139         for subdir in subdirs:
 140             sd = os.path.join(d, subdir)
 141             for root, dirs, files in os.walk(sd, followlinks=False):
 142                 for fn in files:
 143                     if fn.endswith('~'):
 144                         continue
 145                     if fn.endswith('.inst'):
 146                         continue
 147                     ffn = os.path.join(root, fn)
 148                     try:
 149                         s = os.stat(ffn)
 150                     except FileNotFoundError:
 151                         continue
 152                     if not s.st_mode & stat.S_IXUSR:
 153                         continue
 154                     if not (subdir == 'bin' or is_git_file(ffn)):
 155                         continue
 156
 157                     if filename_filter is not None:
 158                         if not filename_filter(ffn):
 159                             continue
 160
 161                     if shebang_filter is not None:
 162                         try:
 163                             f = open(ffn, 'rb')
 164                         except OSError as e:
 165                             print("could not open %s: %s" % (ffn, e))
 166                             continue
 167                         line = f.read(40)
 168                         f.close()
 169                         if not shebang_filter(line):
 170                             continue
 171
 172                     name = safename('_', fn)
 173                     while name in cache:
 174                         name += '_'
 175                     cache[name] = ffn
 176
 177     return cache.items()
 178
 179 # For ELF we only look at /bin/* top level.
 180 def elf_file_name(fn):
 181     fn = fn.partition('bin/')[2]
 182     return fn and '/' not in fn and 'test' not in fn and 'ldb' in fn
 183
 184 def elf_shebang(x):
 185     return x[:4] == b'\x7fELF'
 186
 187 elf_cache = {}
 188 def elf_iterator():
 189     return script_iterator(BASEDIR, elf_cache,
 190                            shebang_filter=elf_shebang,
 191                            filename_filter=elf_file_name,
 192                            subdirs=['bin'])
 193
 194
 195 perl_shebang = re.compile(br'#!.+perl').match
 196
 197 perl_script_cache = {}
 198 def perl_script_iterator():
 199     return script_iterator(BASEDIR, perl_script_cache, perl_shebang)
 200
 201
 202 python_shebang = re.compile(br'#!.+python').match
 203
 204 python_script_cache = {}
 205 def python_script_iterator():
 206     return script_iterator(BASEDIR, python_script_cache, python_shebang)
 207
 208
 209 class PerlScriptUsageTests(TestCase):
 210     """Perl scripts run without arguments should print a usage string,
 211         not fail with a traceback.
 212     """
 213
 214     @classmethod
 215     def initialise(cls):
 216         for name, filename in perl_script_iterator():
 217             print(name, filename)
 218
 219
 220 class PythonScriptUsageTests(TestCase):
 221     """Python scripts run without arguments should print a usage string,
 222         not fail with a traceback.
 223     """
 224
 225     @classmethod
 226     def initialise(cls):
 227         for name, filename in python_script_iterator():
 228             # We add the actual tests after the class definition so we
 229             # can give individual names to them, so we can have a
 230             # knownfail list.
 231             fn = filename.replace(BASEDIR, '').lstrip('/')
 232
 233             if fn in EXCLUDE_USAGE:
 234                 print("skipping %s (EXCLUDE_USAGE)" % filename)
 235                 continue
 236
 237             if os.path.dirname(fn) in EXCLUDE_DIRS:
 238                 print("skipping %s (EXCLUDE_DIRS)" % filename)
 239                 continue
 240
 241             def _f(self, filename=filename):
 242                 print(filename)
 243                 try:
 244                     p = subprocess.Popen(['python3', filename],
 245                                          stderr=subprocess.PIPE,
 246                                          stdout=subprocess.PIPE)
 247                     out, err = p.communicate(timeout=5)
 248                 except OSError as e:
 249                     self.fail("Error: %s" % e)
 250                 except subprocess.SubprocessError as e:
 251                     self.fail("Subprocess error: %s" % e)
 252
 253                 err = err.decode('utf-8')
 254                 out = out.decode('utf-8')
 255                 self.assertNotIn('Traceback', err)
 256
 257                 self.assertIn('usage', out.lower() + err.lower(),
 258                               'stdout:\n%s\nstderr:\n%s' % (out, err))
 259
 260             attr = 'test_%s' % name
 261             if hasattr(cls, attr):
 262                 raise RuntimeError(f'Usage test ‘{attr}’ already exists!')
 263             setattr(cls, attr, _f)
 264
 265
 266 class HelpTestSuper(TestCase):
 267     """Python scripts run with -h or --help should print a help string,
 268     and exit with success.
 269     """
 270     check_return_code = True
 271     check_consistency = True
 272     check_contains_usage = True
 273     check_multiline = True
 274     check_merged_out_and_err = False
 275
 276     interpreter = None
 277
 278     options_start = None
 279     options_end = None
 280     def iterator(self):
 281         raise NotImplementedError("Subclass this "
 282                                   "and add an iterator function!")
 283
 284     @classmethod
 285     def initialise(cls):
 286         for name, filename in cls.iterator():
 287             # We add the actual tests after the class definition so we
 288             # can give individual names to them, so we can have a
 289             # knownfail list.
 290             fn = filename.replace(BASEDIR, '').lstrip('/')
 291
 292             if fn in EXCLUDE_HELP:
 293                 print("skipping %s (EXCLUDE_HELP)" % filename)
 294                 continue
 295
 296             if os.path.dirname(fn) in EXCLUDE_DIRS:
 297                 print("skipping %s (EXCLUDE_DIRS)" % filename)
 298                 continue
 299
 300             def _f(self, filename=filename):
 301                 print(filename)
 302                 for h in ('--help', '-h'):
 303                     cmd = [filename, h]
 304                     if self.interpreter:
 305                         cmd.insert(0, self.interpreter)
 306                     try:
 307                         p = subprocess.Popen(cmd,
 308                                              stderr=subprocess.PIPE,
 309                                              stdout=subprocess.PIPE)
 310                         out, err = p.communicate(timeout=5)
 311                     except OSError as e:
 312                         self.fail("Error: %s" % e)
 313                     except subprocess.SubprocessError as e:
 314                         self.fail("Subprocess error: %s" % e)
 315
 316                     err = err.decode('utf-8')
 317                     out = out.decode('utf-8')
 318                     if self.check_merged_out_and_err:
 319                         out = "%s\n%s" % (out, err)
 320
 321                     outl = out[:500].lower()
 322                     # NOTE:
 323                     # These assertions are heuristics, not policy.
 324                     # If your script fails this test when it shouldn't
 325                     # just add it to EXCLUDE_HELP above or change the
 326                     # heuristic.
 327
 328                     # --help should produce:
 329                     #    * multiple lines of help on stdout (not stderr),
 330                     #    * including a "Usage:" string,
 331                     #    * not contradict itself or repeat options,
 332                     #    * and return success.
 333                     #print(out.encode('utf8'))
 334                     #print(err.encode('utf8'))
 335                     if self.check_consistency:
 336                         errors = check_help_consistency(out,
 337                                                         self.options_start,
 338                                                         self.options_end)
 339                         if errors is not None:
 340                             self.fail(errors)
 341
 342                     if self.check_return_code:
 343                         self.assertEqual(p.returncode, 0,
 344                                          "%s %s\nreturncode should not be %d\n"
 345                                          "err:\n%s\nout:\n%s" %
 346                                          (filename, h, p.returncode, err, out))
 347                     if self.check_contains_usage:
 348                         self.assertIn('usage', outl, 'lacks "Usage:"\n')
 349                     if self.check_multiline:
 350                         self.assertIn('\n', out, 'expected multi-line output')
 351
 352             attr = 'test_%s' % name
 353             if hasattr(cls, attr):
 354                 raise RuntimeError(f'Usage test ‘{attr}’ already exists!')
 355             setattr(cls, attr, _f)
 356
 357
 358 class PythonScriptHelpTests(HelpTestSuper):
 359     """Python scripts run with -h or --help should print a help string,
 360     and exit with success.
 361     """
 362     iterator = python_script_iterator
 363     interpreter = 'python3'
 364
 365
 366 class ElfHelpTests(HelpTestSuper):
 367     """ELF binaries run with -h or --help should print a help string,
 368     and exit with success.
 369     """
 370     iterator = elf_iterator
 371     check_return_code = False
 372     check_merged_out_and_err = True
 373
 374
 375 PerlScriptUsageTests.initialise()
 376 PythonScriptUsageTests.initialise()
 377 PythonScriptHelpTests.initialise()
 378 ElfHelpTests.initialise()