fuzzshark: integrate oss-fuzz targets in CMake
authorPeter Wu <peter@lekensteyn.nl>
Tue, 16 Oct 2018 22:47:20 +0000 (00:47 +0200)
committerAnders Broman <a.broman58@gmail.com>
Sat, 20 Oct 2018 07:00:43 +0000 (07:00 +0000)
The current fuzzshark target built by CMake is not usable for fuzzing.
Address this by adding a new ENABLE_FUZZER option that enables mandatory
instrumentation and libFuzzer linking options for the fuzzshark binary.

Create more CMake targets for specific fuzzing targets such as
fuzzshark_ip and fuzzshark_ip_proto-udp. These targets are not built by
default, either build individual targets or use the all-fuzzers target.

Now these binaries are not specific to oss-fuzz, so move them to a new
directory (perhaps the corpora can be added here in the future).
oss-fuzz build.sh is simplified and reuses the CMake targets.

When OSS_FUZZ is set, it will force static linking with external
libraries and limit parallel linker jobs (maybe not necessary for
Google's oss-fuzz builders, but my 8G/6c VM ran out of memory).

Change-Id: If3ba8f60ea1f5c3bd2131223050a81f9acbce05d
Reviewed-on: https://code.wireshark.org/review/30228
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Anders Broman <a.broman58@gmail.com>
CMakeLists.txt
CMakeOptions.txt
fuzz/CMakeLists.txt [new file with mode: 0644]
fuzz/FuzzerInterface.h [moved from tools/oss-fuzzshark/FuzzerInterface.h with 100% similarity]
fuzz/StandaloneFuzzTargetMain.c [moved from tools/oss-fuzzshark/StandaloneFuzzTargetMain.c with 100% similarity]
fuzz/fuzzshark.c [moved from tools/oss-fuzzshark/fuzzshark.c with 100% similarity]
tools/oss-fuzzshark/build.sh

index f4ec71e5f31bc2c2019e241c555715fd950dcf95..e282941f6be113857e07bad690aea0e1597deac1 100644 (file)
@@ -268,6 +268,36 @@ if(APPLE AND EXISTS /usr/local/opt/qt5)
        set (QT_FIND_PACKAGE_OPTIONS PATHS /usr/local/opt/qt5)
 endif()
 
+set(OSS_FUZZ OFF CACHE BOOL "Whether building for oss-fuzz")
+mark_as_advanced(OSS_FUZZ)
+if(OSS_FUZZ)
+       if(ENABLE_FUZZER)
+               # In oss-fuzz mode, the fuzzing engine can be afl or libFuzzer.
+               message(FATAL_ERROR "Cannot force libFuzzer when using oss-fuzz")
+       endif()
+       # Must not depend on external dependencies so statically link all libs.
+       set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+endif()
+
+#
+# Linking can consume a lot of memory, especially when built with ASAN and
+# static libraries (like oss-fuzz) or Debug mode. With Ninja, the number of
+# parallel linker processes is constrained by job parallelism (-j), but this can
+# be reduced further by setting "job pools" to a lower number.
+#
+if(CMAKE_MAKE_PROGRAM MATCHES "ninja" AND OSS_FUZZ)
+       # Assume oss-fuzz linker jobs do not require more than 1.2G per task
+       set(per_job_memory_mb 1200)
+       cmake_host_system_information(RESULT total_memory_mb QUERY TOTAL_PHYSICAL_MEMORY)
+       math(EXPR parallel_link_jobs "${total_memory_mb} / ${per_job_memory_mb}")
+       if(parallel_link_jobs LESS 1)
+               set(parallel_link_jobs 1)
+       endif()
+       set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${parallel_link_jobs})
+       set(CMAKE_JOB_POOL_LINK link_job_pool)
+       message(STATUS "Ninja job pool size: ${parallel_link_jobs}")
+endif()
+
 # Always enable position-independent code when compiling, even for
 # executables, so you can build position-independent executables.
 # -pie is added below for non-MSVC.
@@ -732,6 +762,21 @@ if(ENABLE_UBSAN)
        set(CMAKE_CXX_FLAGS "-fsanitize=undefined ${CMAKE_CXX_FLAGS}")
 endif()
 
+if(ENABLE_FUZZER)
+       # Available since Clang >= 6
+       # Will enable coverage flags which can be used by the fuzzshark target.
+       cmake_push_check_state()
+       set(CMAKE_REQUIRED_LIBRARIES "-fsanitize=fuzzer-no-link")
+       check_c_compiler_flag(-fsanitize=fuzzer C__fsanitize_fuzzer_no_link_VALID)
+       check_cxx_compiler_flag(-fsanitize=fuzzer CXX__fsanitize_fuzzer_no_link_VALID)
+       cmake_pop_check_state()
+       if(NOT C__fsanitize_fuzzer_no_link_VALID OR NOT CXX__fsanitize_fuzzer_no_link_VALID)
+               message(FATAL_ERROR "ENABLE_FUZZER was requested, but not supported!")
+       endif()
+       set(CMAKE_C_FLAGS "-fsanitize=fuzzer-no-link ${CMAKE_C_FLAGS}")
+       set(CMAKE_CXX_FLAGS "-fsanitize=fuzzer-no-link ${CMAKE_CXX_FLAGS}")
+endif()
+
 set(WERROR_COMMON_FLAGS "")
 if(NOT DISABLE_WERROR AND NOT ENABLE_EXTRA_COMPILER_WARNINGS)
        if(CMAKE_C_COMPILER_ID MATCHES "MSVC")
@@ -2351,20 +2396,8 @@ if(BUILD_randpkt)
        install(TARGETS randpkt RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif()
 
-if(BUILD_fuzzshark)
-       set(fuzzshark_LIBS
-               wiretap
-               ${LIBEPAN_LIBS}
-       )
-       set(fuzzshark_FILES
-               tools/oss-fuzzshark/fuzzshark.c
-               tools/oss-fuzzshark/StandaloneFuzzTargetMain.c
-               version_info.c
-       )
-       add_executable(fuzzshark ${fuzzshark_FILES})
-       add_dependencies(fuzzshark version)
-       set_extra_executable_properties(fuzzshark "Executables")
-       target_link_libraries(fuzzshark ${fuzzshark_LIBS})
+if(BUILD_fuzzshark OR ENABLE_FUZZER OR OSS_FUZZ)
+       add_subdirectory(fuzz)
 endif()
 
 if(BUILD_text2pcap)
index c267c00884adcb5795ab444747135b32a86cc41f..2cfe6419d47bd25ed3328b5b630d74048f03ebf5 100644 (file)
@@ -45,6 +45,7 @@ option(ENABLE_CODE_ANALYSIS "Enable the compiler's static analyzer if possible"
 option(ENABLE_ASAN "Enable AddressSanitizer (ASAN) for debugging (degrades performance)" OFF)
 option(ENABLE_TSAN "Enable ThreadSanitizer (TSan) for debugging" OFF)
 option(ENABLE_UBSAN "Enable UndefinedBehaviorSanitizer (UBSan) for debugging" OFF)
+option(ENABLE_FUZZER "Enable libFuzzer instrumentation for use with fuzzshark" OFF)
 option(ENABLE_CHECKHF_CONFLICT "Enable hf conflict check for debugging (start-up may be slower)" OFF)
 option(ENABLE_CCACHE "Speed up compiling and linking using ccache if possible" OFF)
 
diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt
new file mode 100644 (file)
index 0000000..c572f90
--- /dev/null
@@ -0,0 +1,142 @@
+# CMakeLists.txt
+#
+# Wireshark - Network traffic analyzer
+# By Gerald Combs <gerald@wireshark.org>
+# Copyright 1998 Gerald Combs
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+
+# List of dissectors compiled below, which should be turned off.
+# This is done to avoid single fuzzer (like IP) to call UDP protocols, which can go back to IP, and so on..
+# While doing so might find some bugs, but it's likely to be the problem for too big corpus in oss-fuzzer
+# (see: https://github.com/google/oss-fuzz/issues/1087).
+# + udplite - it's sharing most of code with UDP.
+set(FUZZ_DISABLED_DISSECTORS ip udp udplite ospf bgp dhcp json)
+
+set(FUZZ_DISSECTORS ip)
+set(FUZZ_IP_PROTO_DISSECTORS udp ospf)
+
+set(FUZZ_TCP_PORT_DISSECTORS bgp)
+# list(APPEND FUZZ_TCP_PORT_DISSECTORS bzr)     # disabled, cause of known problem.
+# list(APPEND FUZZ_TCP_PORT_DISSECTORS echo)    # disabled, too simple.
+
+set(FUZZ_UDP_PORT_DISSECTORS dns dhcp)
+# list(FUZZ_UDP_PORT_DISSECTORS bfd)            # disabled, too simple.
+
+set(FUZZ_MEDIA_TYPE_DISSECTORS json)
+
+set(fuzzshark_LIBS
+       wiretap
+       ${LIBEPAN_LIBS}
+)
+if(OSS_FUZZ)
+       if("$ENV{LIB_FUZZING_ENGINE}" STREQUAL "")
+               message(FATAL_ERROR "LIB_FUZZING_ENGINE is not set!")
+       endif()
+       list(APPEND fuzzshark_LIBS $ENV{LIB_FUZZING_ENGINE})
+endif()
+set(fuzzshark_FILES
+       fuzzshark.c
+       ${CMAKE_SOURCE_DIR}/version_info.c
+)
+set(FUZZ_LINK_FLAGS "${WS_LINK_FLAGS}")
+if(ENABLE_FUZZER)
+       set(FUZZ_LINK_FLAGS "${FUZZ_LINK_FLAGS} -fsanitize=fuzzer")
+endif()
+if(OSS_FUZZ)
+       # libFuzzingEngine.a is not position independent, so cannot use -pie.
+       set(FUZZ_LINK_FLAGS "${FUZZ_LINK_FLAGS} -no-pie")
+endif()
+
+# Convert the list of disabled dissectors from a;b;c -> "a", "b", "c"
+# for use in fuzzshark.c (macro)
+string(REGEX REPLACE "([^;]+)" "\"\\1\"" FUZZ_DISABLED_DISSECTORS_MACRO "${FUZZ_DISABLED_DISSECTORS}")
+string(REPLACE ";" ", " FUZZ_DISABLED_DISSECTORS_MACRO "${FUZZ_DISABLED_DISSECTORS_MACRO}")
+
+# Targets that are build via all-fuzzers:
+# - fuzzshark: a non-specific fuzz target, configurable through env vars (requires BUILD_fuzzshark)
+# - fuzzshark_<target>: fuzz target for a specific dissector target.
+# - fuzzshark_<table>-<target>: fuzz target for a specific dissector via a dissector table.
+add_custom_target(all-fuzzers)
+
+function(fuzzshark_set_common_options fuzzer_name)
+       add_dependencies(${fuzzer_name} version)
+       # Sanitizers require a C++ runtime, so use a C++ linker.
+       set_target_properties(${fuzzer_name} PROPERTIES
+               FOLDER "Fuzzers"
+               LINK_FLAGS "${FUZZ_LINK_FLAGS}"
+               LINKER_LANGUAGE "CXX"
+       )
+       target_link_libraries(${fuzzer_name} ${fuzzshark_LIBS})
+       add_dependencies(all-fuzzers ${fuzzer_name})
+endfunction()
+
+if(BUILD_fuzzshark)
+       if(NOT (ENABLE_FUZZER OR OSS_FUZZ))
+               # libFuzzer includes a main routine that enables fuzzing. If
+               # support for fuzzing was not enabled, add a small standalone
+               # target that can be used to test-compile fuzzshark.c.
+               list(APPEND fuzzshark_FILES StandaloneFuzzTargetMain.c)
+       endif()
+       add_executable(fuzzshark ${fuzzshark_FILES})
+       fuzzshark_set_common_options(fuzzshark)
+endif()
+
+# Create a new dissector fuzzer target.
+# If <dissector_table> is empty, <name> will be called directly.
+# If <dissector_table> is non-empty, a dissector with filter name <name> will be
+# looked up in dissector table <dissector_table>.
+function(generate_fuzzer dissector_table name)
+       if(NOT (ENABLE_FUZZER OR OSS_FUZZ))
+               return()
+       endif()
+
+       if(dissector_table STREQUAL "")
+               set(fuzzer_name fuzzshark_${name})
+       else()
+               # "ip.proto" and "udp" -> "ip_proto-udp"
+               set(fuzzer_name fuzzshark_${dissector_table}-${name})
+               string(REPLACE "." "_" fuzzer_name ${fuzzer_name})
+       endif()
+
+       add_executable(${fuzzer_name} EXCLUDE_FROM_ALL ${fuzzshark_FILES})
+       fuzzshark_set_common_options(${fuzzer_name})
+       target_compile_definitions(${fuzzer_name} PRIVATE
+               FUZZ_DISSECTOR_LIST=${FUZZ_DISABLED_DISSECTORS_MACRO}
+               FUZZ_DISSECTOR_TABLE="${dissector_table}"
+               FUZZ_DISSECTOR_TARGET="${name}"
+       )
+endfunction()
+
+# Add fuzzer targets for every dissector in list FUZZ_<table-var>_DISSECTORS,
+# where <table-var> changes a <table> such as "ip.proto" into "IP_PROTO".
+function(add_table_fuzzers table)
+       string(REPLACE "." "_" table_var ${table})
+       string(TOUPPER "${table_var}" table_var)
+       foreach(dissector IN LISTS FUZZ_${table_var}_DISSECTORS)
+               generate_fuzzer(${table} ${dissector})
+       endforeach()
+endfunction()
+
+foreach(dissector IN LISTS FUZZ_DISSECTORS)
+       generate_fuzzer("" ${dissector})
+endforeach()
+
+add_table_fuzzers("ip.proto")
+add_table_fuzzers("tcp.port")
+add_table_fuzzers("udp.port")
+add_table_fuzzers("media_type")
+
+#
+# Editor modelines  -  http://www.wireshark.org/tools/modelines.html
+#
+# Local variables:
+# c-basic-offset: 8
+# tab-width: 8
+# indent-tabs-mode: t
+# End:
+#
+# vi: set shiftwidth=8 tabstop=8 noexpandtab:
+# :indentSize=8:tabSize=8:noTabs=false:
+#
index ff74dda22213872c8eb88fb4012947d2c5f9444b..bc86cdc0576d0a6ac95027a27a5a67ac20cc155b 100755 (executable)
@@ -3,70 +3,20 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-# List of dissectors compiled below, which should be turned off.
-# This is done to avoid single fuzzer (like IP) to call UDP protocols, which can go back to IP, and so on..
-# While doing so might find some bugs, but it's likely to be the problem for too big corpus in oss-fuzzer
-# (see: https://github.com/google/oss-fuzz/issues/1087).
-# + udplite - it's sharing most of code with UDP.
-DISSECTOR_LIST='"ip", "udp", "udplite", "ospf", "bgp", "dhcp", "json"'
-
-FUZZ_DISSECTORS="ip"
-
-FUZZ_IP_PROTO_DISSECTORS="udp ospf"
-
-FUZZ_TCP_PORT_DISSECTORS="bgp"
-# FUZZ_TCP_PORT_DISSECTORS="$FUZZ_TCP_PORT_DISSECTORS bzr"   # disabled, cause of known problem.
-# FUZZ_TCP_PORT_DISSECTORS="$FUZZ_TCP_PORT_DISSECTORS echo"  # disabled, too simple.
-
-FUZZ_UDP_PORT_DISSECTORS="dns dhcp"
-# FUZZ_UDP_PORT_DISSECTORS="$FUZZ_UDP_PORT_DISSECTORS bfd"   # disabled, too simple.
-
-FUZZ_MEDIA_TYPE_DISSECTORS="json"
-
 # TODO: support specifing targets in args. Google oss-fuzz specifies 'all'.
 
-# generate_fuzzer <fuzzer_target> <fuzzer_cflags>
-generate_fuzzer()
-{
-  local fuzzer_target="$1" fuzzer_cflags="$2" fuzzer_name
-
-  fuzzer_name="fuzzshark_$1"
+# TODO update oss-fuzz configuration to build with OSS_FUZZ=1? This is necessary
+# to build the fuzzshark_* targets for oss-fuzz.
+cmake -DOSS_FUZZ=1 .
 
-  $CC $CFLAGS -I $WIRESHARK_INSTALL_PATH/include/wireshark/ `pkg-config --cflags glib-2.0` \
-      $SRC/wireshark/tools/oss-fuzzshark/fuzzshark.c \
-      -c -o $WORK/${fuzzer_name}.o \
-      $fuzzer_cflags -DFUZZ_DISSECTOR_LIST="$DISSECTOR_LIST"
-
-  $CXX $CXXFLAGS $WORK/${fuzzer_name}.o \
-      -o $OUT/${fuzzer_name} \
-      ${WIRESHARK_FUZZERS_COMMON_FLAGS}
+cmake --build . --target all-fuzzers
 
+for file in run/fuzzshark_*; do
+  fuzzer_name="${file##*/}"
+  fuzzer_target="${fuzzer_name#fuzzshark_}"
+  mv "$file" "$OUT/"
   echo -en "[libfuzzer]\nmax_len = 1024\n" > $OUT/${fuzzer_name}.options
   if [ -d "$SAMPLES_DIR/${fuzzer_target}" ]; then
     zip -j $OUT/${fuzzer_name}_seed_corpus.zip $SAMPLES_DIR/${fuzzer_target}/*/*.bin
   fi
-}
-
-WIRESHARK_FUZZERS_COMMON_FLAGS="-lFuzzingEngine \
-    -L"$WIRESHARK_INSTALL_PATH/lib" -lwireshark -lwiretap -lwsutil \
-    -Wl,-Bstatic `pkg-config --libs glib-2.0` -pthread -lpcre -lgcrypt -lgpg-error -lz -Wl,-Bdynamic"
-
-for dissector in $FUZZ_DISSECTORS; do
-  generate_fuzzer "${dissector}" -DFUZZ_DISSECTOR_TARGET=\"$dissector\"
-done
-
-for dissector in $FUZZ_IP_PROTO_DISSECTORS; do
-  generate_fuzzer "ip_proto-${dissector}" "-DFUZZ_DISSECTOR_TABLE=\"ip.proto\" -DFUZZ_DISSECTOR_TARGET=\"$dissector\""
-done
-
-for dissector in $FUZZ_TCP_PORT_DISSECTORS; do
-  generate_fuzzer "tcp_port-${dissector}" "-DFUZZ_DISSECTOR_TABLE=\"tcp.port\" -DFUZZ_DISSECTOR_TARGET=\"$dissector\""
-done
-
-for dissector in $FUZZ_UDP_PORT_DISSECTORS; do
-  generate_fuzzer "udp_port-${dissector}" "-DFUZZ_DISSECTOR_TABLE=\"udp.port\" -DFUZZ_DISSECTOR_TARGET=\"$dissector\""
-done
-
-for dissector in $FUZZ_MEDIA_TYPE_DISSECTORS; do
-  generate_fuzzer "media_type-${dissector}" "-DFUZZ_DISSECTOR_TABLE=\"media_type\" -DFUZZ_DISSECTOR_TARGET=\"$dissector\""
 done