python/samba/tests/usage.py

   1 # Unix SMB/CIFS implementation.
   2 # Copyright © Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import sys
  19 import subprocess
  20 from samba.tests import TestCase, check_help_consistency
  21 from unittest import TestSuite
  22 import re
  23 import stat
  24
  25 if 'SRCDIR_ABS' in os.environ:
  26     BASEDIR = os.environ['SRCDIR_ABS']
  27 else:
  28     BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
  29                                            '../../..'))
  30
  31 TEST_DIRS = [
  32     "bootstrap",
  33     "testdata",
  34     "ctdb",
  35     "dfs_server",
  36     "pidl",
  37     "auth",
  38     "packaging",
  39     "python",
  40     "include",
  41     "nsswitch",
  42     "libcli",
  43     "coverity",
  44     "release-scripts",
  45     "testprogs",
  46     "bin",
  47     "source3",
  48     "docs-xml",
  49     "buildtools",
  50     "file_server",
  51     "dynconfig",
  52     "source4",
  53     "tests",
  54     "libds",
  55     "selftest",
  56     "lib",
  57     "script",
  58     "traffic",
  59     "testsuite",
  60     "libgpo",
  61     "wintest",
  62     "librpc",
  63 ]
  64
  65
  66 EXCLUDE_USAGE = {
  67     'script/autobuild.py',  # defaults to mount /memdisk/
  68     'script/bisect-test.py',
  69     'ctdb/utils/etcd/ctdb_etcd_lock',
  70     'selftest/filter-subunit',
  71     'selftest/format-subunit',
  72     'bin/gen_output.py',  # too much output!
  73     'source4/scripting/bin/gen_output.py',
  74     'lib/ldb/tests/python/index.py',
  75     'lib/ldb/tests/python/api.py',
  76     'source4/selftest/tests.py',
  77     'buildtools/bin/waf',
  78     'selftest/tap2subunit',
  79     'script/show_test_time',
  80     'source4/scripting/bin/subunitrun',
  81     'bin/samba_downgrade_db',
  82     'source4/scripting/bin/samba_downgrade_db',
  83     'source3/selftest/tests.py',
  84     'selftest/tests.py',
  85     'python/samba/subunit/run.py',
  86     'bin/python/samba/subunit/run.py',
  87     'python/samba/tests/dcerpc/raw_protocol.py',
  88     'python/samba/tests/smb-notify.py',
  89     'python/samba/tests/krb5/kcrypto.py',
  90     'python/samba/tests/krb5/simple_tests.py',
  91     'python/samba/tests/krb5/s4u_tests.py',
  92     'python/samba/tests/krb5/xrealm_tests.py',
  93     'python/samba/tests/krb5/as_canonicalization_tests.py',
  94     'python/samba/tests/krb5/compatability_tests.py',
  95     'python/samba/tests/krb5/rfc4120_constants.py',
  96     'python/samba/tests/krb5/kdc_tests.py',
  97     'python/samba/tests/krb5/kdc_base_test.py',
  98     'python/samba/tests/krb5/kdc_tgs_tests.py',
  99     'python/samba/tests/krb5/test_ccache.py',
 100     'python/samba/tests/krb5/test_ldap.py',
 101     'python/samba/tests/krb5/test_rpc.py',
 102     'python/samba/tests/krb5/test_smb.py',
 103     'python/samba/tests/krb5/ms_kile_client_principal_lookup_tests.py',
 104     'python/samba/tests/krb5/as_req_tests.py',
 105     'python/samba/tests/krb5/fast_tests.py',
 106     'python/samba/tests/krb5/rodc_tests.py',
 107     'python/samba/tests/krb5/salt_tests.py',
 108     'python/samba/tests/krb5/spn_tests.py',
 109     'python/samba/tests/krb5/alias_tests.py',
 110     'python/samba/tests/krb5/test_min_domain_uid.py',
 111     'python/samba/tests/krb5/test_idmap_nss.py',
 112     'python/samba/tests/krb5/pac_align_tests.py',
 113     'python/samba/tests/krb5/protected_users_tests.py',
 114     'python/samba/tests/krb5/nt_hash_tests.py',
 115     'python/samba/tests/krb5/kpasswd_tests.py',
 116     'python/samba/tests/krb5/claims_tests.py',
 117     'python/samba/tests/krb5/lockout_tests.py',
 118     'python/samba/tests/krb5/group_tests.py',
 119     'lib/compression/tests/scripts/three-byte-hash',
 120     'python/samba/tests/krb5/etype_tests.py',
 121     'python/samba/tests/krb5/device_tests.py',
 122     'python/samba/tests/krb5/claims_in_pac.py',
 123     'python/samba/tests/krb5/authn_policy_tests.py',
 124 }
 125
 126 EXCLUDE_HELP = {
 127     'selftest/tap2subunit',
 128     'wintest/test-s3.py',
 129     'wintest/test-s4-howto.py',
 130 }
 131
 132
 133 EXCLUDE_DIRS = {
 134     'source3/script/tests',
 135     'python/examples',
 136     'source4/dsdb/tests/python',
 137     'bin/ab',
 138     'bin/python/samba/tests',
 139     'bin/python/samba/tests/dcerpc',
 140     'bin/python/samba/tests/krb5',
 141     'python/samba/tests/bin',
 142 }
 143
 144
 145 def _init_git_file_finder():
 146     """Generate a function that quickly answers the question:
 147     'is this a git file?'
 148     """
 149     git_file_cache = set()
 150     p = subprocess.run(['git',
 151                         '-C', BASEDIR,
 152                         'ls-files',
 153                         '-z'],
 154                        stdout=subprocess.PIPE)
 155     if p.returncode == 0:
 156         for fn in p.stdout.split(b'\0'):
 157             git_file_cache.add(os.path.join(BASEDIR, fn.decode('utf-8')))
 158     return git_file_cache.__contains__
 159
 160
 161 is_git_file = _init_git_file_finder()
 162
 163
 164 def script_iterator(d=BASEDIR, cache=None,
 165                     shebang_filter=None,
 166                     filename_filter=None,
 167                     subdirs=None):
 168     if subdirs is None:
 169         subdirs = TEST_DIRS
 170     if not cache:
 171         safename = re.compile(r'\W+').sub
 172         for subdir in subdirs:
 173             sd = os.path.join(d, subdir)
 174             for root, dirs, files in os.walk(sd, followlinks=False):
 175                 for fn in files:
 176                     if fn.endswith('~'):
 177                         continue
 178                     if fn.endswith('.inst'):
 179                         continue
 180                     ffn = os.path.join(root, fn)
 181                     try:
 182                         s = os.stat(ffn)
 183                     except FileNotFoundError:
 184                         continue
 185                     if not s.st_mode & stat.S_IXUSR:
 186                         continue
 187                     if not (subdir == 'bin' or is_git_file(ffn)):
 188                         continue
 189
 190                     if filename_filter is not None:
 191                         if not filename_filter(ffn):
 192                             continue
 193
 194                     if shebang_filter is not None:
 195                         try:
 196                             f = open(ffn, 'rb')
 197                         except OSError as e:
 198                             print("could not open %s: %s" % (ffn, e))
 199                             continue
 200                         line = f.read(40)
 201                         f.close()
 202                         if not shebang_filter(line):
 203                             continue
 204
 205                     name = safename('_', fn)
 206                     while name in cache:
 207                         name += '_'
 208                     cache[name] = ffn
 209
 210     return cache.items()
 211
 212 # For ELF we only look at /bin/* top level.
 213 def elf_file_name(fn):
 214     fn = fn.partition('bin/')[2]
 215     return fn and '/' not in fn and 'test' not in fn and 'ldb' in fn
 216
 217 def elf_shebang(x):
 218     return x[:4] == b'\x7fELF'
 219
 220 elf_cache = {}
 221 def elf_iterator():
 222     return script_iterator(BASEDIR, elf_cache,
 223                            shebang_filter=elf_shebang,
 224                            filename_filter=elf_file_name,
 225                            subdirs=['bin'])
 226
 227
 228 perl_shebang = re.compile(br'#!.+perl').match
 229
 230 perl_script_cache = {}
 231 def perl_script_iterator():
 232     return script_iterator(BASEDIR, perl_script_cache, perl_shebang)
 233
 234
 235 python_shebang = re.compile(br'#!.+python').match
 236
 237 python_script_cache = {}
 238 def python_script_iterator():
 239     return script_iterator(BASEDIR, python_script_cache, python_shebang)
 240
 241
 242 class PerlScriptUsageTests(TestCase):
 243     """Perl scripts run without arguments should print a usage string,
 244         not fail with a traceback.
 245     """
 246
 247     @classmethod
 248     def initialise(cls):
 249         for name, filename in perl_script_iterator():
 250             print(name, filename)
 251
 252
 253 class PythonScriptUsageTests(TestCase):
 254     """Python scripts run without arguments should print a usage string,
 255         not fail with a traceback.
 256     """
 257
 258     @classmethod
 259     def initialise(cls):
 260         for name, filename in python_script_iterator():
 261             # We add the actual tests after the class definition so we
 262             # can give individual names to them, so we can have a
 263             # knownfail list.
 264             fn = filename.replace(BASEDIR, '').lstrip('/')
 265
 266             if fn in EXCLUDE_USAGE:
 267                 print("skipping %s (EXCLUDE_USAGE)" % filename)
 268                 continue
 269
 270             if os.path.dirname(fn) in EXCLUDE_DIRS:
 271                 print("skipping %s (EXCLUDE_DIRS)" % filename)
 272                 continue
 273
 274             def _f(self, filename=filename):
 275                 print(filename)
 276                 try:
 277                     p = subprocess.Popen(['python3', filename],
 278                                          stderr=subprocess.PIPE,
 279                                          stdout=subprocess.PIPE)
 280                     out, err = p.communicate(timeout=5)
 281                 except OSError as e:
 282                     self.fail("Error: %s" % e)
 283                 except subprocess.SubprocessError as e:
 284                     self.fail("Subprocess error: %s" % e)
 285
 286                 err = err.decode('utf-8')
 287                 out = out.decode('utf-8')
 288                 self.assertNotIn('Traceback', err)
 289
 290                 self.assertIn('usage', out.lower() + err.lower(),
 291                               'stdout:\n%s\nstderr:\n%s' % (out, err))
 292
 293             setattr(cls, 'test_%s' % name, _f)
 294
 295
 296 class HelpTestSuper(TestCase):
 297     """Python scripts run with -h or --help should print a help string,
 298     and exit with success.
 299     """
 300     check_return_code = True
 301     check_consistency = True
 302     check_contains_usage = True
 303     check_multiline = True
 304     check_merged_out_and_err = False
 305
 306     interpreter = None
 307
 308     options_start = None
 309     options_end = None
 310     def iterator(self):
 311         raise NotImplementedError("Subclass this "
 312                                   "and add an iterator function!")
 313
 314     @classmethod
 315     def initialise(cls):
 316         for name, filename in cls.iterator():
 317             # We add the actual tests after the class definition so we
 318             # can give individual names to them, so we can have a
 319             # knownfail list.
 320             fn = filename.replace(BASEDIR, '').lstrip('/')
 321
 322             if fn in EXCLUDE_HELP:
 323                 print("skipping %s (EXCLUDE_HELP)" % filename)
 324                 continue
 325
 326             if os.path.dirname(fn) in EXCLUDE_DIRS:
 327                 print("skipping %s (EXCLUDE_DIRS)" % filename)
 328                 continue
 329
 330             def _f(self, filename=filename):
 331                 print(filename)
 332                 for h in ('--help', '-h'):
 333                     cmd = [filename, h]
 334                     if self.interpreter:
 335                         cmd.insert(0, self.interpreter)
 336                     try:
 337                         p = subprocess.Popen(cmd,
 338                                              stderr=subprocess.PIPE,
 339                                              stdout=subprocess.PIPE)
 340                         out, err = p.communicate(timeout=5)
 341                     except OSError as e:
 342                         self.fail("Error: %s" % e)
 343                     except subprocess.SubprocessError as e:
 344                         self.fail("Subprocess error: %s" % e)
 345
 346                     err = err.decode('utf-8')
 347                     out = out.decode('utf-8')
 348                     if self.check_merged_out_and_err:
 349                         out = "%s\n%s" % (out, err)
 350
 351                     outl = out[:500].lower()
 352                     # NOTE:
 353                     # These assertions are heuristics, not policy.
 354                     # If your script fails this test when it shouldn't
 355                     # just add it to EXCLUDE_HELP above or change the
 356                     # heuristic.
 357
 358                     # --help should produce:
 359                     #    * multiple lines of help on stdout (not stderr),
 360                     #    * including a "Usage:" string,
 361                     #    * not contradict itself or repeat options,
 362                     #    * and return success.
 363                     #print(out.encode('utf8'))
 364                     #print(err.encode('utf8'))
 365                     if self.check_consistency:
 366                         errors = check_help_consistency(out,
 367                                                         self.options_start,
 368                                                         self.options_end)
 369                         if errors is not None:
 370                             self.fail(errors)
 371
 372                     if self.check_return_code:
 373                         self.assertEqual(p.returncode, 0,
 374                                          "%s %s\nreturncode should not be %d\n"
 375                                          "err:\n%s\nout:\n%s" %
 376                                          (filename, h, p.returncode, err, out))
 377                     if self.check_contains_usage:
 378                         self.assertIn('usage', outl, 'lacks "Usage:"\n')
 379                     if self.check_multiline:
 380                         self.assertIn('\n', out, 'expected multi-line output')
 381
 382             setattr(cls, 'test_%s' % name, _f)
 383
 384
 385 class PythonScriptHelpTests(HelpTestSuper):
 386     """Python scripts run with -h or --help should print a help string,
 387     and exit with success.
 388     """
 389     iterator = python_script_iterator
 390     interpreter = 'python3'
 391
 392
 393 class ElfHelpTests(HelpTestSuper):
 394     """ELF binaries run with -h or --help should print a help string,
 395     and exit with success.
 396     """
 397     iterator = elf_iterator
 398     check_return_code = False
 399     check_merged_out_and_err = True
 400
 401
 402 PerlScriptUsageTests.initialise()
 403 PythonScriptUsageTests.initialise()
 404 PythonScriptHelpTests.initialise()
 405 ElfHelpTests.initialise()