From bf680707acfdadcd6301657448dcf3bd8c8fa60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0tetiar?= Date: Sun, 8 Dec 2019 13:12:47 +0100 Subject: [PATCH] tests: add unit tests covered with Clang sanitizers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we run all tests via Valgrind. This patch adds 2nd batch of tests which are compiled with Clang AddressSanitizer[1], LeakSanitizer[2] and UndefinedBehaviorSanitizer[3] in order to catch more issues during QA on CI. AddressSanitizer is a fast memory error detector. The tool can detect the following types of bugs: * Out-of-bounds accesses to heap, stack and globals * Use-after-free, use-after-return, use-after-scope * Double-free, invalid free LeakSanitizer is a run-time memory leak detector. It can be combined with AddressSanitizer to get both memory error and leak detection, or used in a stand-alone mode. UndefinedBehaviorSanitizer (UBSan) is a fast undefined behavior detector. UBSan modifies the program at compile-time to catch various kinds of undefined behavior during program execution, for example: * Using misaligned or null pointer * Signed integer overflow * Conversion to, from, or between floating-point types which would overflow the destination 1. http://clang.llvm.org/docs/AddressSanitizer.html 2. http://http://clang.llvm.org/docs/LeakSanitizer.html 3. http://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html Signed-off-by: Petr Štetiar --- CMakeLists.txt | 12 +++ tests/CMakeLists.txt | 5 +- tests/cram/test_avl.t | 9 +++ tests/cram/test_base64.t | 30 ++++++- tests/cram/test_blobmsg.t | 15 ++++ tests/cram/test_jshn.t | 144 ++++++++++++++++++++++++++++++++++ tests/cram/test_json_script.t | 55 +++++++++++++ tests/cram/test_list.t | 20 +++++ tests/cram/test_runqueue.t | 12 +++ 9 files changed, 297 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b0c9e3..dcd455c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,14 @@ INSTALL(TARGETS ubox ubox-static ADD_SUBDIRECTORY(lua) ADD_SUBDIRECTORY(examples) +MACRO(ADD_UNIT_TEST_SAN name) + ADD_EXECUTABLE(${name}-san ${name}.c) + TARGET_COMPILE_OPTIONS(${name}-san PRIVATE -g -fno-omit-frame-pointer -fsanitize=undefined,address,leak -fno-sanitize-recover=all) + TARGET_LINK_OPTIONS(${name}-san PRIVATE -fsanitize=undefined,address,leak) + TARGET_LINK_LIBRARIES(${name}-san ubox blobmsg_json json_script ${json}) + TARGET_INCLUDE_DIRECTORIES(${name}-san PRIVATE ${PROJECT_SOURCE_DIR}) +ENDMACRO(ADD_UNIT_TEST_SAN) + IF(UNIT_TESTING) ENABLE_TESTING() ADD_SUBDIRECTORY(tests) @@ -62,6 +70,10 @@ IF(EXISTS ${json}) SET_TARGET_PROPERTIES(blobmsg_json-static PROPERTIES OUTPUT_NAME blobmsg_json) + IF(UNIT_TESTING) + ADD_UNIT_TEST_SAN(jshn) + ENDIF(UNIT_TESTING) + ADD_EXECUTABLE(jshn jshn.c) TARGET_LINK_LIBRARIES(jshn blobmsg_json ${json}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1c44825..bd22057 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,7 @@ ENDMACRO(ADD_UNIT_TEST) FILE(GLOB test_cases "test-*.c") FOREACH(test_case ${test_cases}) - GET_FILENAME_COMPONENT(test_case ${test_case} NAME_WE) - ADD_UNIT_TEST(${test_case}) + GET_FILENAME_COMPONENT(test_case ${test_case} NAME_WE) + ADD_UNIT_TEST(${test_case}) + ADD_UNIT_TEST_SAN(${test_case}) ENDFOREACH(test_case) diff --git a/tests/cram/test_avl.t b/tests/cram/test_avl.t index 19a8d21..d8d1640 100644 --- a/tests/cram/test_avl.t +++ b/tests/cram/test_avl.t @@ -9,3 +9,12 @@ check that avl is producing expected results: test_basics: delete 'one' element test_basics: for each element reverse: zero two twelve three ten six seven nine four five eleven eight test_basics: delete all elements + + $ test-avl-san + test_basics: insert: 0=zero 0=one 0=two 0=three 0=four 0=five 0=six 0=seven 0=eight 0=nine 0=ten 0=eleven 0=twelve + test_basics: insert duplicate: -1=zero -1=one -1=two -1=three -1=four -1=five -1=six -1=seven -1=eight -1=nine -1=ten -1=eleven -1=twelve + test_basics: first=eight last=zero + test_basics: for each element: eight eleven five four nine one seven six ten three twelve two zero + test_basics: delete 'one' element + test_basics: for each element reverse: zero two twelve three ten six seven nine four five eleven eight + test_basics: delete all elements diff --git a/tests/cram/test_base64.t b/tests/cram/test_base64.t index 0a7a9d5..ade41fb 100644 --- a/tests/cram/test_base64.t +++ b/tests/cram/test_base64.t @@ -20,14 +20,38 @@ check that base64 is producing expected results: 5 fooba 6 foobar + $ test-b64-san + 0 + 4 Zg== + 4 Zm8= + 4 Zm9v + 8 Zm9vYg== + 8 Zm9vYmE= + 8 Zm9vYmFy + 0 + 1 f + 2 fo + 3 foo + 4 foob + 5 fooba + 6 foobar + check that b64_encode and b64_decode assert invalid input - $ alias check="egrep '(dumped|Assertion)' | sed 's;.*\(b64_.*code\).*\(Assertion.*$\);\1: \2;' | LC_ALL=C sort" + $ alias check="egrep '(dumped|Assertion)' output.log | sed 's;.*\(b64_.*code\).*\(Assertion.*$\);\1: \2;' | LC_ALL=C sort" - $ test-b64_decode 2>&1 | check + $ test-b64_decode 2> output.log; check Aborted (core dumped) b64_decode: Assertion `dest && targsize > 0' failed. - $ test-b64_encode 2>&1 | check + $ test-b64_encode 2> output.log; check + Aborted (core dumped) + b64_encode: Assertion `dest && targsize > 0' failed. + + $ test-b64_decode-san 2> output.log; check + Aborted (core dumped) + b64_decode: Assertion `dest && targsize > 0' failed. + + $ test-b64_encode-san 2> output.log; check Aborted (core dumped) b64_encode: Assertion `dest && targsize > 0' failed. diff --git a/tests/cram/test_blobmsg.t b/tests/cram/test_blobmsg.t index 504a056..3a5801a 100644 --- a/tests/cram/test_blobmsg.t +++ b/tests/cram/test_blobmsg.t @@ -15,3 +15,18 @@ check that blobmsg is producing expected results: \tworld : 2 (esc) } json: {"message":"Hello, world!","testdata":{"double":133.700000,"hello":1,"world":"2"},"list":[0,1,2,133.700000]} + + $ test-blobmsg-san + Message: Hello, world! + List: { + 0 + 1 + 2 + 133.700000 + } + Testdata: { + \tdouble : 133.700000 (esc) + \thello : 1 (esc) + \tworld : 2 (esc) + } + json: {"message":"Hello, world!","testdata":{"double":133.700000,"hello":1,"world":"2"},"list":[0,1,2,133.700000]} diff --git a/tests/cram/test_jshn.t b/tests/cram/test_jshn.t index 1881a3d..b2f2853 100644 --- a/tests/cram/test_jshn.t +++ b/tests/cram/test_jshn.t @@ -9,12 +9,20 @@ check usage: Usage: jshn [-n] [-i] -r |-R |-o |-p |-w [2] + $ jshn-san + Usage: jshn-san [-n] [-i] -r |-R |-o |-p |-w + [2] + test bad json: $ jshn -r '[]' Failed to parse message data [1] + $ jshn-san -r '[]' + Failed to parse message data + [1] + test good json: $ jshn -r '{"foo": "bar", "baz": {"next": "meep"}}' @@ -24,16 +32,31 @@ test good json: json_add_string 'next' 'meep'; json_close_object; + $ jshn-san -r '{"foo": "bar", "baz": {"next": "meep"}}' + json_init; + json_add_string 'foo' 'bar'; + json_add_object 'baz'; + json_add_string 'next' 'meep'; + json_close_object; + test json from file: $ echo '[]' > test.json; jshn -R test.json Failed to parse message data [1] + $ echo '[]' > test.json; jshn-san -R test.json + Failed to parse message data + [1] + $ jshn -R nada.json Error opening nada.json [3] + $ jshn-san -R nada.json + Error opening nada.json + [3] + $ echo '{"foo": "bar", "baz": {"next": "meep"}}' > test.json; jshn -R test.json json_init; json_add_string 'foo' 'bar'; @@ -41,38 +64,74 @@ test json from file: json_add_string 'next' 'meep'; json_close_object; + $ echo '{"foo": "bar", "baz": {"next": "meep"}}' > test.json; jshn-san -R test.json + json_init; + json_add_string 'foo' 'bar'; + json_add_object 'baz'; + json_add_string 'next' 'meep'; + json_close_object; + test json formatting without prepared environment: $ jshn -p procd -w { } + $ jshn-san -p procd -w + { } + $ jshn -i -p procd -w { \t (esc) } + $ jshn-san -i -p procd -w + { + \t (esc) + } + $ jshn -i -n -p procd -w { \t (esc) } (no-eol) + $ jshn-san -i -n -p procd -w + { + \t (esc) + } (no-eol) + $ jshn -p procd -o test.json; cat test.json { } + $ jshn-san -p procd -o test.json; cat test.json + { } + $ jshn -i -p procd -o test.json; cat test.json { \t (esc) } + $ jshn-san -i -p procd -o test.json; cat test.json + { + \t (esc) + } + $ jshn -i -n -p procd -o test.json; cat test.json { \t (esc) } (no-eol) + $ jshn-san -i -n -p procd -o test.json; cat test.json + { + \t (esc) + } (no-eol) + $ chmod oug= test.json $ jshn -i -n -p procd -o test.json Error opening test.json [3] + $ jshn-san -i -n -p procd -o test.json + Error opening test.json + [3] $ rm -f test.json test json formatting with prepared environment: @@ -104,6 +163,9 @@ test json formatting with prepared environment: $ jshn -p procd -w { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } } + $ jshn-san -p procd -w + { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } } + $ jshn -i -p procd -w { \t"name": "urngd", (esc) @@ -123,6 +185,25 @@ test json formatting with prepared environment: \t} (esc) } + $ jshn-san -i -p procd -w + { + \t"name": "urngd", (esc) + \t"script": "/etc/init.d/urngd", (esc) + \t"instances": { (esc) + \t\t"instance1": { (esc) + \t\t\t"command": [ (esc) + \t\t\t\t"/sbin/urngd" (esc) + \t\t\t] (esc) + \t\t} (esc) + \t}, (esc) + \t"triggers": [ (esc) + \t\t (esc) + \t], (esc) + \t"data": { (esc) + \t\t (esc) + \t} (esc) + } + $ jshn -n -i -p procd -w { \t"name": "urngd", (esc) @@ -142,9 +223,31 @@ test json formatting with prepared environment: \t} (esc) } (no-eol) + $ jshn-san -n -i -p procd -w + { + \t"name": "urngd", (esc) + \t"script": "/etc/init.d/urngd", (esc) + \t"instances": { (esc) + \t\t"instance1": { (esc) + \t\t\t"command": [ (esc) + \t\t\t\t"/sbin/urngd" (esc) + \t\t\t] (esc) + \t\t} (esc) + \t}, (esc) + \t"triggers": [ (esc) + \t\t (esc) + \t], (esc) + \t"data": { (esc) + \t\t (esc) + \t} (esc) + } (no-eol) + $ jshn -p procd -o test.json; cat test.json { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } } + $ jshn-san -p procd -o test.json; cat test.json + { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } } + $ jshn -i -p procd -o test.json; cat test.json { \t"name": "urngd", (esc) @@ -164,6 +267,25 @@ test json formatting with prepared environment: \t} (esc) } + $ jshn-san -i -p procd -o test.json; cat test.json + { + \t"name": "urngd", (esc) + \t"script": "/etc/init.d/urngd", (esc) + \t"instances": { (esc) + \t\t"instance1": { (esc) + \t\t\t"command": [ (esc) + \t\t\t\t"/sbin/urngd" (esc) + \t\t\t] (esc) + \t\t} (esc) + \t}, (esc) + \t"triggers": [ (esc) + \t\t (esc) + \t], (esc) + \t"data": { (esc) + \t\t (esc) + \t} (esc) + } + $ jshn -n -i -p procd -o test.json; cat test.json { \t"name": "urngd", (esc) @@ -183,7 +305,29 @@ test json formatting with prepared environment: \t} (esc) } (no-eol) + $ jshn-san -n -i -p procd -o test.json; cat test.json + { + \t"name": "urngd", (esc) + \t"script": "/etc/init.d/urngd", (esc) + \t"instances": { (esc) + \t\t"instance1": { (esc) + \t\t\t"command": [ (esc) + \t\t\t\t"/sbin/urngd" (esc) + \t\t\t] (esc) + \t\t} (esc) + \t}, (esc) + \t"triggers": [ (esc) + \t\t (esc) + \t], (esc) + \t"data": { (esc) + \t\t (esc) + \t} (esc) + } (no-eol) + $ chmod oug= test.json $ jshn -n -i -p procd -o test.json Error opening test.json [3] + $ jshn-san -n -i -p procd -o test.json + Error opening test.json + [3] diff --git a/tests/cram/test_json_script.t b/tests/cram/test_json_script.t index 3e80a5c..4af7f54 100644 --- a/tests/cram/test_json_script.t +++ b/tests/cram/test_json_script.t @@ -3,6 +3,7 @@ set test bin path: $ [ -n "$TEST_BIN_DIR" ] && export PATH="$TEST_BIN_DIR:$PATH" $ export TEST_INPUTS="$TESTDIR/inputs" $ alias js="valgrind --quiet --leak-check=full test-json-script" + $ alias js-san="test-json-script-san" check that json-script is producing expected results: @@ -10,25 +11,46 @@ check that json-script is producing expected results: Usage: test-json-script [VARNAME=value] [254] + $ js-san + Usage: test-json-script-san [VARNAME=value] + [254] + $ echo '}' > test.json; js test.json load JSON data from test.json failed. + $ echo '}' > test.json; js-san test.json + load JSON data from test.json failed. + $ js nada.json 2>&1 | grep load.*failed load JSON data from nada.json failed. + $ js-san nada.json 2>&1 | grep load.*failed + load JSON data from nada.json failed. + $ echo '[ [ ] [ ] ]' > test.json; js test.json load JSON data from test.json failed. + $ echo '[ [ ] [ ] ]' > test.json; js-san test.json + load JSON data from test.json failed. + check example json-script: $ js $TEST_INPUTS/json-script.json exec /%/ exec_if_or + $ js-san $TEST_INPUTS/json-script.json + exec /%/ + exec_if_or + $ js EXECVAR=meh ORVAR=meep $TEST_INPUTS/json-script.json exec meh /%/ exec_if_or meep + $ js-san EXECVAR=meh ORVAR=meep $TEST_INPUTS/json-script.json + exec meh /%/ + exec_if_or meep + check has expression: $ echo ' @@ -43,12 +65,21 @@ check has expression: $ js VAR=foo test.json echo bar + $ js-san VAR=foo test.json + echo bar + $ js VAR=bar test.json echo bar + $ js-san VAR=bar test.json + echo bar + $ js test.json echo baz + $ js-san test.json + echo baz + check eq expression: $ echo ' @@ -63,12 +94,21 @@ check eq expression: $ js VAR=bar test.json echo foo + $ js-san VAR=bar test.json + echo foo + $ js VAR=xxx test.json echo baz + $ js-san VAR=xxx test.json + echo baz + $ js test.json echo baz + $ js-san test.json + echo baz + check regex single expression: $ echo ' @@ -83,14 +123,29 @@ check regex single expression: $ js VAR=hello test.json echo bar + $ js-san VAR=hello test.json + echo bar + $ js VAR=.ell. test.json echo bar + $ js-san VAR=.ell. test.json + echo bar + $ js test.json echo baz + $ js-san test.json + echo baz + $ js VAR= test.json echo baz + $ js-san VAR= test.json + echo baz + $ js VAR=hell test.json echo baz + + $ js-san VAR=hell test.json + echo baz diff --git a/tests/cram/test_list.t b/tests/cram/test_list.t index f7f18bd..81affad 100644 --- a/tests/cram/test_list.t +++ b/tests/cram/test_list.t @@ -20,3 +20,23 @@ check that list is producing expected results: test_basics: list_for_each_entry_reverse: one eleven ten nine eight seven six five four three two test_basics: delete all entries test_basics: list_empty: yes + + $ test-list-san + test_basics: list_empty: yes + test_basics: list_add_tail: zero one two three four five six seven eight nine ten eleven twelve + test_basics: list_empty: no + test_basics: first=zero last=twelve + test_basics: 'zero' is first, yes + test_basics: 'twelve' is last, yes + test_basics: removing 'twelve' and 'zero' + test_basics: first=one last=eleven + test_basics: 'one' is first, yes + test_basics: 'eleven' is last, yes + test_basics: moving 'one' to the tail + test_basics: first=two last=one + test_basics: 'two' is first, yes + test_basics: 'one' is last, yes + test_basics: list_for_each_entry: two three four five six seven eight nine ten eleven one + test_basics: list_for_each_entry_reverse: one eleven ten nine eight seven six five four three two + test_basics: delete all entries + test_basics: list_empty: yes diff --git a/tests/cram/test_runqueue.t b/tests/cram/test_runqueue.t index 4d49110..227f414 100644 --- a/tests/cram/test_runqueue.t +++ b/tests/cram/test_runqueue.t @@ -12,3 +12,15 @@ check that runqueue is producing expected results: [1/1] cancel 'sleep 1' [0/1] finish 'sleep 1' All done! + + $ test-runqueue-san + [1/1] start 'sleep 1' + [1/1] cancel 'sleep 1' + [0/1] finish 'sleep 1' + [1/1] start 'sleep 1' + [1/1] cancel 'sleep 1' + [0/1] finish 'sleep 1' + [1/1] start 'sleep 1' + [1/1] cancel 'sleep 1' + [0/1] finish 'sleep 1' + All done!