tests: add unit tests covered with Clang sanitizers

Currently we run all tests via Valgrind. This patch adds 2nd batch of tests which are compiled with Clang AddressSanitizer[1], LeakSanitizer[2] and UndefinedBehaviorSanitizer[3] in order to catch more issues during QA on CI. AddressSanitizer is a fast memory error detector. The tool can detect the following types of bugs: * Out-of-bounds accesses to heap, stack and globals * Use-after-free, use-after-return, use-after-scope * Double-free, invalid free LeakSanitizer is a run-time memory leak detector. It can be combined with AddressSanitizer to get both memory error and leak detection, or used in a stand-alone mode. UndefinedBehaviorSanitizer (UBSan) is a fast undefined behavior detector. UBSan modifies the program at compile-time to catch various kinds of undefined behavior during program execution, for example: * Using misaligned or null pointer * Signed integer overflow * Conversion to, from, or between floating-point types which would overflow the destination 1. http://clang.llvm.org/docs/AddressSanitizer.html 2. http://http://clang.llvm.org/docs/LeakSanitizer.html 3. http://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html Signed-off-by: Petr Štetiar <ynezz@true.cz>
2019-12-08 13:12:47 +01:00 · 2019-12-08 13:12:47 +01:00 · bf680707ac
commit bf680707ac
parent f804578847
9 changed files with 297 additions and 5 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -48,6 +48,14 @@ INSTALL(TARGETS ubox ubox-static
 ADD_SUBDIRECTORY(lua)
 ADD_SUBDIRECTORY(examples)

+MACRO(ADD_UNIT_TEST_SAN name)
+  ADD_EXECUTABLE(${name}-san ${name}.c)
+  TARGET_COMPILE_OPTIONS(${name}-san PRIVATE -g -fno-omit-frame-pointer -fsanitize=undefined,address,leak -fno-sanitize-recover=all)
+  TARGET_LINK_OPTIONS(${name}-san PRIVATE -fsanitize=undefined,address,leak)
+  TARGET_LINK_LIBRARIES(${name}-san ubox blobmsg_json json_script ${json})
+  TARGET_INCLUDE_DIRECTORIES(${name}-san PRIVATE ${PROJECT_SOURCE_DIR})
+ENDMACRO(ADD_UNIT_TEST_SAN)
+
 IF(UNIT_TESTING)
  ENABLE_TESTING()
  ADD_SUBDIRECTORY(tests)
@ -62,6 +70,10 @@ IF(EXISTS ${json})
 	SET_TARGET_PROPERTIES(blobmsg_json-static
 			      PROPERTIES OUTPUT_NAME blobmsg_json)

+	IF(UNIT_TESTING)
+		ADD_UNIT_TEST_SAN(jshn)
+	ENDIF(UNIT_TESTING)
+
 	ADD_EXECUTABLE(jshn jshn.c)
 	TARGET_LINK_LIBRARIES(jshn blobmsg_json ${json})

--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@ -8,6 +8,7 @@ ENDMACRO(ADD_UNIT_TEST)

 FILE(GLOB test_cases "test-*.c")
 FOREACH(test_case ${test_cases})
-	GET_FILENAME_COMPONENT(test_case ${test_case} NAME_WE)
-	ADD_UNIT_TEST(${test_case})
+  GET_FILENAME_COMPONENT(test_case ${test_case} NAME_WE)
+  ADD_UNIT_TEST(${test_case})
+  ADD_UNIT_TEST_SAN(${test_case})
 ENDFOREACH(test_case)
--- a/tests/cram/test_avl.t
+++ b/tests/cram/test_avl.t
@ -9,3 +9,12 @@ check that avl is producing expected results:
  test_basics: delete 'one' element
  test_basics: for each element reverse: zero two twelve three ten six seven nine four five eleven eight 
  test_basics: delete all elements
+
+  $ test-avl-san
+  test_basics: insert: 0=zero 0=one 0=two 0=three 0=four 0=five 0=six 0=seven 0=eight 0=nine 0=ten 0=eleven 0=twelve 
+  test_basics: insert duplicate: -1=zero -1=one -1=two -1=three -1=four -1=five -1=six -1=seven -1=eight -1=nine -1=ten -1=eleven -1=twelve 
+  test_basics: first=eight last=zero
+  test_basics: for each element: eight eleven five four nine one seven six ten three twelve two zero 
+  test_basics: delete 'one' element
+  test_basics: for each element reverse: zero two twelve three ten six seven nine four five eleven eight 
+  test_basics: delete all elements
--- a/tests/cram/test_base64.t
+++ b/tests/cram/test_base64.t
@ -20,14 +20,38 @@ check that base64 is producing expected results:
  5 fooba
  6 foobar

+  $ test-b64-san
+  0 
+  4 Zg==
+  4 Zm8=
+  4 Zm9v
+  8 Zm9vYg==
+  8 Zm9vYmE=
+  8 Zm9vYmFy
+  0 
+  1 f
+  2 fo
+  3 foo
+  4 foob
+  5 fooba
+  6 foobar
+
 check that b64_encode and b64_decode assert invalid input

-  $ alias check="egrep '(dumped|Assertion)' | sed 's;.*\(b64_.*code\).*\(Assertion.*$\);\1: \2;' | LC_ALL=C sort"
+  $ alias check="egrep '(dumped|Assertion)' output.log | sed 's;.*\(b64_.*code\).*\(Assertion.*$\);\1: \2;' | LC_ALL=C sort"

-  $ test-b64_decode 2>&1 | check
+  $ test-b64_decode 2> output.log; check
  Aborted (core dumped)
  b64_decode: Assertion `dest && targsize > 0' failed.

-  $ test-b64_encode 2>&1 | check
+  $ test-b64_encode 2> output.log; check
+  Aborted (core dumped)
+  b64_encode: Assertion `dest && targsize > 0' failed.
+
+  $ test-b64_decode-san 2> output.log; check
+  Aborted (core dumped)
+  b64_decode: Assertion `dest && targsize > 0' failed.
+
+  $ test-b64_encode-san 2> output.log; check
  Aborted (core dumped)
  b64_encode: Assertion `dest && targsize > 0' failed.
--- a/tests/cram/test_blobmsg.t
+++ b/tests/cram/test_blobmsg.t
@ -15,3 +15,18 @@ check that blobmsg is producing expected results:
  \tworld : 2 (esc)
  }
  json: {"message":"Hello, world!","testdata":{"double":133.700000,"hello":1,"world":"2"},"list":[0,1,2,133.700000]}
+
+  $ test-blobmsg-san
+  Message: Hello, world!
+  List: {
+  0
+  1
+  2
+  133.700000
+  }
+  Testdata: {
+  \tdouble : 133.700000 (esc)
+  \thello : 1 (esc)
+  \tworld : 2 (esc)
+  }
+  json: {"message":"Hello, world!","testdata":{"double":133.700000,"hello":1,"world":"2"},"list":[0,1,2,133.700000]}
--- a/tests/cram/test_jshn.t
+++ b/tests/cram/test_jshn.t
@ -9,12 +9,20 @@ check usage:
  Usage: jshn [-n] [-i] -r <message>|-R <file>|-o <file>|-p <prefix>|-w
  [2]

+  $ jshn-san
+  Usage: jshn-san [-n] [-i] -r <message>|-R <file>|-o <file>|-p <prefix>|-w
+  [2]
+
 test bad json:

  $ jshn -r '[]'
  Failed to parse message data
  [1]

+  $ jshn-san -r '[]'
+  Failed to parse message data
+  [1]
+
 test good json:

  $ jshn -r '{"foo": "bar", "baz": {"next": "meep"}}'
@ -24,16 +32,31 @@ test good json:
  json_add_string 'next' 'meep';
  json_close_object;

+  $ jshn-san -r '{"foo": "bar", "baz": {"next": "meep"}}'
+  json_init;
+  json_add_string 'foo' 'bar';
+  json_add_object 'baz';
+  json_add_string 'next' 'meep';
+  json_close_object;
+
 test json from file:

  $ echo '[]' > test.json; jshn -R test.json
  Failed to parse message data
  [1]

+  $ echo '[]' > test.json; jshn-san -R test.json
+  Failed to parse message data
+  [1]
+
  $ jshn -R nada.json
  Error opening nada.json
  [3]

+  $ jshn-san -R nada.json
+  Error opening nada.json
+  [3]
+
  $ echo '{"foo": "bar", "baz": {"next": "meep"}}' > test.json; jshn -R test.json
  json_init;
  json_add_string 'foo' 'bar';
@ -41,38 +64,74 @@ test json from file:
  json_add_string 'next' 'meep';
  json_close_object;

+  $ echo '{"foo": "bar", "baz": {"next": "meep"}}' > test.json; jshn-san -R test.json
+  json_init;
+  json_add_string 'foo' 'bar';
+  json_add_object 'baz';
+  json_add_string 'next' 'meep';
+  json_close_object;
+
 test json formatting without prepared environment:

  $ jshn -p procd -w
  { }

+  $ jshn-san -p procd -w
+  { }
+
  $ jshn -i -p procd -w
  {
  \t (esc)
  }

+  $ jshn-san -i -p procd -w
+  {
+  \t (esc)
+  }
+
  $ jshn -i -n -p procd -w
  {
  \t (esc)
  } (no-eol)

+  $ jshn-san -i -n -p procd -w
+  {
+  \t (esc)
+  } (no-eol)
+
  $ jshn -p procd -o test.json; cat test.json
  { }

+  $ jshn-san -p procd -o test.json; cat test.json
+  { }
+
  $ jshn -i -p procd -o test.json; cat test.json
  {
  \t (esc)
  }

+  $ jshn-san -i -p procd -o test.json; cat test.json
+  {
+  \t (esc)
+  }
+
  $ jshn -i -n -p procd -o test.json; cat test.json
  {
  \t (esc)
  } (no-eol)

+  $ jshn-san -i -n -p procd -o test.json; cat test.json
+  {
+  \t (esc)
+  } (no-eol)
+
  $ chmod oug= test.json
  $ jshn -i -n -p procd -o test.json
  Error opening test.json
  [3]
+  $ jshn-san -i -n -p procd -o test.json
+  Error opening test.json
+  [3]
  $ rm -f test.json

 test json formatting with prepared environment:
@ -104,6 +163,9 @@ test json formatting with prepared environment:
  $ jshn -p procd -w
  { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }

+  $ jshn-san -p procd -w
+  { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }
+
  $ jshn -i -p procd -w
  {
  \t"name": "urngd", (esc)
@ -123,6 +185,25 @@ test json formatting with prepared environment:
  \t} (esc)
  }

+  $ jshn-san -i -p procd -w
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  }
+
  $ jshn -n -i -p procd -w
  {
  \t"name": "urngd", (esc)
@ -142,9 +223,31 @@ test json formatting with prepared environment:
  \t} (esc)
  } (no-eol)

+  $ jshn-san -n -i -p procd -w
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  } (no-eol)
+
  $ jshn -p procd -o test.json; cat test.json
  { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }

+  $ jshn-san -p procd -o test.json; cat test.json
+  { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }
+
  $ jshn -i -p procd -o test.json; cat test.json
  {
  \t"name": "urngd", (esc)
@ -164,6 +267,25 @@ test json formatting with prepared environment:
  \t} (esc)
  }

+  $ jshn-san -i -p procd -o test.json; cat test.json
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  }
+
  $ jshn -n -i -p procd -o test.json; cat test.json
  {
  \t"name": "urngd", (esc)
@ -183,7 +305,29 @@ test json formatting with prepared environment:
  \t} (esc)
  } (no-eol)

+  $ jshn-san -n -i -p procd -o test.json; cat test.json
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  } (no-eol)
+
  $ chmod oug= test.json
  $ jshn -n -i -p procd -o test.json
  Error opening test.json
  [3]
+  $ jshn-san -n -i -p procd -o test.json
+  Error opening test.json
+  [3]
--- a/tests/cram/test_json_script.t
+++ b/tests/cram/test_json_script.t
@ -3,6 +3,7 @@ set test bin path:
  $ [ -n "$TEST_BIN_DIR" ] && export PATH="$TEST_BIN_DIR:$PATH"
  $ export TEST_INPUTS="$TESTDIR/inputs"
  $ alias js="valgrind --quiet --leak-check=full test-json-script"
+  $ alias js-san="test-json-script-san"

 check that json-script is producing expected results:

@ -10,25 +11,46 @@ check that json-script is producing expected results:
  Usage: test-json-script [VARNAME=value] <filename_json_script>
  [254]

+  $ js-san
+  Usage: test-json-script-san [VARNAME=value] <filename_json_script>
+  [254]
+
  $ echo '}' > test.json; js test.json
  load JSON data from test.json failed.

+  $ echo '}' > test.json; js-san test.json
+  load JSON data from test.json failed.
+
  $ js nada.json 2>&1 | grep load.*failed
  load JSON data from nada.json failed.

+  $ js-san nada.json 2>&1 | grep load.*failed
+  load JSON data from nada.json failed.
+
  $ echo '[ [ ] [ ] ]' > test.json; js test.json
  load JSON data from test.json failed.

+  $ echo '[ [ ] [ ] ]' > test.json; js-san test.json
+  load JSON data from test.json failed.
+
 check example json-script:

  $ js $TEST_INPUTS/json-script.json
  exec  /%/
  exec_if_or 

+  $ js-san $TEST_INPUTS/json-script.json
+  exec  /%/
+  exec_if_or 
+
  $ js EXECVAR=meh ORVAR=meep $TEST_INPUTS/json-script.json
  exec meh /%/
  exec_if_or meep

+  $ js-san EXECVAR=meh ORVAR=meep $TEST_INPUTS/json-script.json
+  exec meh /%/
+  exec_if_or meep
+
 check has expression:

  $ echo '
@ -43,12 +65,21 @@ check has expression:
  $ js VAR=foo test.json
  echo bar

+  $ js-san VAR=foo test.json
+  echo bar
+
  $ js VAR=bar test.json
  echo bar

+  $ js-san VAR=bar test.json
+  echo bar
+
  $ js test.json
  echo baz

+  $ js-san test.json
+  echo baz
+
 check eq expression:

  $ echo '
@ -63,12 +94,21 @@ check eq expression:
  $ js VAR=bar test.json
  echo foo

+  $ js-san VAR=bar test.json
+  echo foo
+
  $ js VAR=xxx test.json
  echo baz

+  $ js-san VAR=xxx test.json
+  echo baz
+
  $ js test.json
  echo baz

+  $ js-san test.json
+  echo baz
+
 check regex single expression:

  $ echo '
@ -83,14 +123,29 @@ check regex single expression:
  $ js VAR=hello test.json
  echo bar

+  $ js-san VAR=hello test.json
+  echo bar
+
  $ js VAR=.ell. test.json
  echo bar

+  $ js-san VAR=.ell. test.json
+  echo bar
+
  $ js test.json
  echo baz

+  $ js-san test.json
+  echo baz
+
  $ js VAR= test.json
  echo baz

+  $ js-san VAR= test.json
+  echo baz
+
  $ js VAR=hell test.json
  echo baz
+
+  $ js-san VAR=hell test.json
+  echo baz
--- a/tests/cram/test_list.t
+++ b/tests/cram/test_list.t
@ -20,3 +20,23 @@ check that list is producing expected results:
  test_basics: list_for_each_entry_reverse: one eleven ten nine eight seven six five four three two 
  test_basics: delete all entries
  test_basics: list_empty: yes
+
+  $ test-list-san
+  test_basics: list_empty: yes
+  test_basics: list_add_tail: zero one two three four five six seven eight nine ten eleven twelve 
+  test_basics: list_empty: no
+  test_basics: first=zero last=twelve
+  test_basics: 'zero' is first, yes
+  test_basics: 'twelve' is last, yes
+  test_basics: removing 'twelve' and 'zero'
+  test_basics: first=one last=eleven
+  test_basics: 'one' is first, yes
+  test_basics: 'eleven' is last, yes
+  test_basics: moving 'one' to the tail
+  test_basics: first=two last=one
+  test_basics: 'two' is first, yes
+  test_basics: 'one' is last, yes
+  test_basics: list_for_each_entry: two three four five six seven eight nine ten eleven one 
+  test_basics: list_for_each_entry_reverse: one eleven ten nine eight seven six five four three two 
+  test_basics: delete all entries
+  test_basics: list_empty: yes
--- a/tests/cram/test_runqueue.t
+++ b/tests/cram/test_runqueue.t
@ -12,3 +12,15 @@ check that runqueue is producing expected results:
  [1/1] cancel 'sleep 1'
  [0/1] finish 'sleep 1'
  All done!
+
+  $ test-runqueue-san
+  [1/1] start 'sleep 1'
+  [1/1] cancel 'sleep 1'
+  [0/1] finish 'sleep 1'
+  [1/1] start 'sleep 1'
+  [1/1] cancel 'sleep 1'
+  [0/1] finish 'sleep 1'
+  [1/1] start 'sleep 1'
+  [1/1] cancel 'sleep 1'
+  [0/1] finish 'sleep 1'
+  All done!