Browse code

Add unit testing support via cmocka

cmocka [1,2] is a testing framework for C. Adding unit test
capabilities to the openvpn repository will greatly ease the
task of writing correct code.

cmocka source code is added as git submodule in ./vendor. A
submodule approach has been chosen over a classical library
dependency because libcmocka is not available, or only
available in very old versions (e.g. on Ubuntu).

cmocka is build during 'make check' and installed in vendor/dist/.

[1] https://cmocka.org/
[2] https://lwn.net/Articles/558106/

Signed-off-by: Jens Neuhalfen <jens@neuhalfen.name>
Acked-by: Steffan Karger <steffan@karger.me>
Message-Id: <20160525175756.56186-2-openvpn-devel@neuhalfen.name>
URL: http://article.gmane.org/gmane.network.openvpn.devel/11725
Signed-off-by: David Sommerseth <dazo@privateinternetaccess.com>

Jens Neuhalfen authored on 2016/05/26 02:57:55
Showing 15 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,4 @@
0
+[submodule "vendor/cmocka"]
1
+	path = vendor/cmocka
2
+	url = git://git.cryptomilk.org/projects/cmocka.git
3
+	branch = master
... ...
@@ -54,7 +54,7 @@ BUILT_SOURCES = \
54 54
 	config-version.h
55 55
 endif
56 56
 
57
-SUBDIRS = build distro include src sample doc tests
57
+SUBDIRS = build distro include src sample doc vendor tests
58 58
 
59 59
 dist_doc_DATA = \
60 60
 	README \
... ...
@@ -1198,6 +1198,19 @@ sampledir="\$(docdir)/sample"
1198 1198
 AC_SUBST([plugindir])
1199 1199
 AC_SUBST([sampledir])
1200 1200
 
1201
+VENDOR_SRC_ROOT="\$(abs_top_srcdir)/vendor/"
1202
+VENDOR_DIST_ROOT="\$(abs_top_builddir)/vendor/dist"
1203
+VENDOR_BUILD_ROOT="\$(abs_top_builddir)/vendor/.build"
1204
+AC_SUBST([VENDOR_SRC_ROOT])
1205
+AC_SUBST([VENDOR_BUILD_ROOT])
1206
+AC_SUBST([VENDOR_DIST_ROOT])
1207
+
1208
+TEST_LDFLAGS="-lcmocka -L\$(abs_top_builddir)/vendor/dist/lib -Wl,-rpath,\$(abs_top_builddir)/vendor/dist/lib"
1209
+TEST_CFLAGS="-I\$(top_srcdir)/include -I\$(abs_top_builddir)/vendor/dist/include"
1210
+
1211
+AC_SUBST([TEST_LDFLAGS])
1212
+AC_SUBST([TEST_CFLAGS])
1213
+
1201 1214
 AC_CONFIG_FILES([
1202 1215
 	version.sh
1203 1216
 	Makefile
... ...
@@ -1216,6 +1229,9 @@ AC_CONFIG_FILES([
1216 1216
 	src/plugins/auth-pam/Makefile
1217 1217
 	src/plugins/down-root/Makefile
1218 1218
 	tests/Makefile
1219
+        tests/unit_tests/Makefile
1220
+        tests/unit_tests/example_test/Makefile
1221
+        vendor/Makefile
1219 1222
 	sample/Makefile
1220 1223
 	doc/Makefile
1221 1224
 ])
... ...
@@ -12,6 +12,8 @@
12 12
 MAINTAINERCLEANFILES = \
13 13
 	$(srcdir)/Makefile.in
14 14
 
15
+SUBDIRS = unit_tests
16
+
15 17
 test_scripts = t_client.sh t_lpback.sh t_cltsrv.sh
16 18
 
17 19
 TESTS_ENVIRONMENT = top_srcdir="$(top_srcdir)"
... ...
@@ -20,4 +22,3 @@ TESTS = $(test_scripts)
20 20
 dist_noinst_SCRIPTS = \
21 21
 	$(test_scripts) \
22 22
 	t_cltsrv-down.sh
23
-
24 23
new file mode 100644
... ...
@@ -0,0 +1 @@
0
+*_testdriver
0 1
new file mode 100644
... ...
@@ -0,0 +1,3 @@
0
+AUTOMAKE_OPTIONS = foreign
1
+
2
+SUBDIRS = example_test
0 3
new file mode 100644
... ...
@@ -0,0 +1,40 @@
0
+Unit Tests
1
+===========
2
+
3
+This directory contains unit tests for openvpn. New features/bugfixes should be written in a test friendly way and come with corresponding tests.
4
+
5
+Run tests
6
+----------
7
+
8
+Tests are run by `make check`. A failed tests stops test execution. To run all
9
+tests regardless of errors call `make -k check`.
10
+
11
+Add new tests to existing test suite
12
+-------------------------------------
13
+
14
+Test suites are organized in directories. [example_test/](example_test/) is an example
15
+for a test suite with two test executables. Feel free to use it as a template for new tests.
16
+
17
+Test suites
18
+--------------------
19
+
20
+Test suites live inside a subdirectory of `$ROOT/tests/unit_tests`, e.g. `$ROOT/tests/unit_tests/my_feature`.
21
+
22
+Test suites are configured by a `Makefile.am`. Tests are executed by testdrivers. One testsuite can contain more than one testdriver.
23
+
24
+### Hints
25
+* Name suites & testdrivers in a way that the name of the driver says something about which component/feature is tested
26
+* Name the testdriver executable `*_testdriver`. This way it gets picked up by the default `.gitignore`
27
+  * If this is not feasible: Add all output to a `.gitignore`* Use descriptive test names: `coffee_brewing__with_no_beans__fails` vs. `test34`
28
+* Testing a configurable feature?  Wrap test execution with a conditional (see [auth_pam](plugins/auth-pam/Makefile.am) for an example)
29
+* Add multiple test-drivers when one testdriver looks crowded with tests
30
+
31
+### New Test Suites
32
+1.  Organize tests in folders for features.
33
+2.  Add the new test directory to `SUBDIRS` in `Makefile.am`
34
+3.  Edit `configure.ac` and add the new `Makefile` to `AC_CONFIG_FILES`
35
+4.  Run `./configure`, and *enable* the feature you'd like to test
36
+5.  Make sure that `make check` runs your tests
37
+6.  Check: Would a stranger be able to easily find your tests by you looking at the test output?
38
+7. Run `./configure`, and *disable* the feature you'd like to test
39
+8.  Make sure that `make check` does *not run* your tests
0 40
new file mode 100644
... ...
@@ -0,0 +1,13 @@
0
+AUTOMAKE_OPTIONS = foreign
1
+
2
+check_PROGRAMS = example_testdriver example2_testdriver
3
+
4
+TESTS = $(check_PROGRAMS)
5
+
6
+example_testdriver_CFLAGS  = @TEST_CFLAGS@
7
+example_testdriver_LDFLAGS = @TEST_LDFLAGS@
8
+example_testdriver_SOURCES = test.c
9
+
10
+example2_testdriver_CFLAGS  = @TEST_CFLAGS@
11
+example2_testdriver_LDFLAGS = @TEST_LDFLAGS@
12
+example2_testdriver_SOURCES = test2.c
0 13
new file mode 100644
... ...
@@ -0,0 +1,3 @@
0
+This test only checks that test compilation works. This example contains two test executables.
1
+
2
+These tests can be used as template for 'real' tests.
0 3
new file mode 100644
... ...
@@ -0,0 +1,46 @@
0
+#include <stdio.h>
1
+#include <unistd.h>
2
+#include <stdlib.h>
3
+#include <stdarg.h>
4
+#include <string.h>
5
+#include <setjmp.h>
6
+#include <cmocka.h>
7
+
8
+static int setup(void **state) {
9
+     int *answer  = malloc(sizeof(int));
10
+
11
+     *answer=42;
12
+     *state=answer;
13
+
14
+     return 0;
15
+}
16
+
17
+static int teardown(void **state) {
18
+     free(*state);
19
+
20
+     return 0;
21
+}
22
+
23
+static void null_test_success(void **state) {
24
+    (void) state;
25
+}
26
+
27
+static void int_test_success(void **state) {
28
+     int *answer = *state;
29
+     assert_int_equal(*answer, 42);
30
+}
31
+
32
+static void failing_test(void **state) {
33
+     // This tests fails to test that make check fails
34
+     assert_int_equal(0, 42);
35
+}
36
+
37
+int main(void) {
38
+    const struct CMUnitTest tests[] = {
39
+        cmocka_unit_test(null_test_success),
40
+        cmocka_unit_test_setup_teardown(int_test_success, setup, teardown),
41
+//        cmocka_unit_test(failing_test),
42
+    };
43
+
44
+    return cmocka_run_group_tests_name("success_test", tests, NULL, NULL);
45
+}
0 46
new file mode 100644
... ...
@@ -0,0 +1,21 @@
0
+#include <stdio.h>
1
+#include <unistd.h>
2
+#include <stdlib.h>
3
+#include <stdarg.h>
4
+#include <string.h>
5
+#include <setjmp.h>
6
+#include <cmocka.h>
7
+
8
+
9
+static void test_true(void **state) {
10
+    (void) state;
11
+}
12
+
13
+
14
+int main(void) {
15
+    const struct CMUnitTest tests[] = {
16
+        cmocka_unit_test(test_true),
17
+    };
18
+
19
+    return cmocka_run_group_tests_name("success_test2", tests, NULL, NULL);
20
+}
0 21
new file mode 100644
... ...
@@ -0,0 +1,2 @@
0
+.build/
1
+dist/
0 2
new file mode 100644
... ...
@@ -0,0 +1,24 @@
0
+AUTOMAKE_OPTIONS = foreign
1
+
2
+cmockasrc     = @VENDOR_SRC_ROOT@/cmocka  # needs an absolute path bc. of the cmake invocation
3
+cmockabuild   = @VENDOR_BUILD_ROOT@/cmocka
4
+cmockainstall = @VENDOR_DIST_ROOT@
5
+
6
+MAINTAINERCLEANFILES = \
7
+	$(srcdir)/Makefile.in \
8
+	$(cmockabuild) \
9
+	$(cmockainstall) \
10
+	@VENDOR_BUILD_ROOT@
11
+
12
+distdir:
13
+	mkdir -p $(cmockainstall)
14
+
15
+libcmocka: distdir
16
+	mkdir -p $(cmockabuild)
17
+	(cd $(cmockabuild) && cmake -DCMAKE_INSTALL_PREFIX=$(cmockainstall) $(cmockasrc) && make && make install)
18
+
19
+check: libcmocka
20
+
21
+clean:
22
+	rm -rf $(cmockabuild)
23
+	rm -rf $(cmockainstall)
0 24
new file mode 100644
... ...
@@ -0,0 +1,8 @@
0
+Vendor
1
+========
2
+
3
+Vendor source libraries are included in this directory. Libraries are included
4
+when there is no good way to ensure that the package is available on all
5
+systems.
6
+
7
+`Makefile.am` compiles these libraries and installs them into ./dist.
0 8
new file mode 160000
... ...
@@ -0,0 +1 @@
0
+Subproject commit b2732b52202ae48f866a024c633466efdbb8e85a