Test+CMake: Make our test grouping more fine-grained.
authorGerald Combs <gerald@wireshark.org>
Mon, 18 Jun 2018 19:01:01 +0000 (12:01 -0700)
committerGerald Combs <gerald@wireshark.org>
Mon, 18 Jun 2018 19:36:26 +0000 (19:36 +0000)
Split our tests by suite_*.group_* instead of suite_*. There are quite a
few dfilter tests and this should make them more parallelizable.

Change-Id: I52371409618cda70dc99811e8de1fb1ad9d9a3b6
Reviewed-on: https://code.wireshark.org/review/28329
Petri-Dish: Gerald Combs <gerald@wireshark.org>
Tested-by: Petri Dish Buildbot
Reviewed-by: Gerald Combs <gerald@wireshark.org>
CMakeLists.txt
test/config.py
test/suite_unittests.py
test/test.py

index 751f8e1..6aaedaa 100644 (file)
@@ -2914,13 +2914,28 @@ set_target_properties(test-programs PROPERTIES
 enable_testing()
 # We could try to build this list dynamically, but given that we tend to
 # go years between adding suites just run
-#     test/test.py --list-suites | sort
+#     test/test.py --list-groups | sort
 # and paste the output here.
-set(_test_suite_list
+set(_test_group_list
        suite_capture
        suite_clopts
        suite_decryption
-       suite_dfilter
+       suite_dfilter.group_bytes_ether
+       suite_dfilter.group_bytes_ipv6
+       suite_dfilter.group_bytes_type
+       suite_dfilter.group_double
+       suite_dfilter.group_integer
+       suite_dfilter.group_integer_1byte
+       suite_dfilter.group_ipv4
+       suite_dfilter.group_membership
+       suite_dfilter.group_range_method
+       suite_dfilter.group_scanner
+       suite_dfilter.group_string_type
+       suite_dfilter.group_stringz
+       suite_dfilter.group_time_relative
+       suite_dfilter.group_time_type
+       suite_dfilter.group_tvb
+       suite_dfilter.group_uint64
        suite_dissection
        suite_fileformats
        suite_io
@@ -2931,9 +2946,9 @@ set(_test_suite_list
        suite_wslua
 )
 if(WIN32)
-       set(_test_suite_program_path ./run/$<CONFIG>)
+       set(_test_group_program_path ./run/$<CONFIG>)
 else()
-       set(_test_suite_program_path ./run)
+       set(_test_group_program_path ./run)
 endif()
 
 # We don't currently handle spaces in arguments. On Windows this
@@ -2942,17 +2957,17 @@ endif()
 set(TEST_EXTRA_ARGS "" CACHE STRING "Extra arguments to pass to test/test.py")
 separate_arguments(TEST_EXTRA_ARGS)
 
-foreach(_suite_name ${_test_suite_list})
+foreach(_group_name ${_test_group_list})
        add_test(
-               NAME ${_suite_name}
+               NAME ${_group_name}
                COMMAND ${CMAKE_COMMAND} -E env PYTHONIOENCODING=UTF-8
                        ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test/test.py
                        --verbose
-                       --program-path ${_test_suite_program_path}
+                       --program-path ${_test_group_program_path}
                        ${TEST_EXTRA_ARGS}
-                       ${_suite_name}
+                       ${_group_name}
                )
-       set_tests_properties(${_suite_name} PROPERTIES TIMEOUT 600)
+       set_tests_properties(${_group_name} PROPERTIES TIMEOUT 600)
 endforeach()
 
 if (GIT_EXECUTABLE)
index cb36de2..e927f65 100644 (file)
@@ -64,7 +64,7 @@ key_dir = os.path.join(this_dir, 'keys')
 lua_dir = os.path.join(this_dir, 'lua')
 tools_dir = os.path.join(this_dir, '..', 'tools')
 
-all_suites = []
+all_groups = []
 
 def canCapture():
     # XXX This appears to be evaluated at the wrong time when called
index 6d896d6..1233807 100644 (file)
@@ -55,21 +55,21 @@ class case_unittests(subprocesstest.SubprocessTestCase):
     def test_unit_ctest_coverage(self):
         '''Make sure CTest runs all of our tests.'''
         with open(os.path.join(config.this_dir, '..', 'CMakeLists.txt')) as cml_fd:
-            suite_re = re.compile('set *\( *_test_suite_list')
+            group_re = re.compile('set *\( *_test_group_list')
             in_list = False
-            cml_suites = []
+            cml_groups = []
             for cml_line in cml_fd:
-                if suite_re.search(cml_line):
+                if group_re.search(cml_line):
                     in_list = True
                     continue
                 if in_list:
                     if ')' in cml_line:
                         break
-                    cml_suites.append(cml_line.strip())
-        cml_suites.sort()
-        if not config.all_suites == cml_suites:
-            diff = '\n'.join(list(difflib.unified_diff(config.all_suites, cml_suites, 'all test suites', 'CMakeLists.txt test suites')))
-            self.fail("CMakeLists.txt doesn't test all available suites:\n" + diff)
+                    cml_groups.append(cml_line.strip())
+        cml_groups.sort()
+        if not config.all_groups == cml_groups:
+            diff = '\n'.join(list(difflib.unified_diff(config.all_groups, cml_groups, 'all test groups', 'CMakeLists.txt test groups')))
+            self.fail("CMakeLists.txt doesn't test all available groups:\n" + diff)
 
 
 class Proto:
index cc6c134..a1c0b05 100755 (executable)
@@ -50,7 +50,8 @@ def main():
     list_group = parser.add_mutually_exclusive_group()
     list_group.add_argument('-l', '--list', action='store_true', help='List tests. One of "all" or a full or partial test name.')
     list_group.add_argument('--list-suites', action='store_true', help='List all suites.')
-    list_group.add_argument('--list-cases', action='store_true', help='List all suites and cases.')
+    list_group.add_argument('--list-groups', action='store_true', help='List all suites and groups.')
+    list_group.add_argument('--list-cases', action='store_true', help='List all suites, groups, and cases.')
     parser.add_argument('-v', '--verbose', action='store_const', const=2, default=1, help='Verbose tests.')
     parser.add_argument('tests_to_run', nargs='*', metavar='test', default=['all'], help='Tests to run. One of "all" or a full or partial test name. Default is "all".')
     args = parser.parse_args()
@@ -94,10 +95,24 @@ def main():
     config.all_suites = list(all_suites)
     config.all_suites.sort()
 
+    all_groups = set()
+    for aid in all_ids:
+        aparts = aid.split('.')
+        if aparts[1].startswith('group_'):
+            all_groups |= {'.'.join(aparts[:2])}
+        else:
+            all_groups |= {aparts[0]}
+    config.all_groups = list(all_groups)
+    config.all_groups.sort()
+
     if args.list_suites:
         print('\n'.join(config.all_suites))
         sys.exit(0)
 
+    if args.list_groups:
+        print('\n'.join(config.all_groups))
+        sys.exit(0)
+
     if args.list_cases:
         cases = set()
         for rid in run_ids: