From 0d2b85fcefb116a9271897d54cf1e878c97db755 Mon Sep 17 00:00:00 2001 From: Trevor Bentley Date: Fri, 30 Jun 2017 09:58:25 +0200 Subject: [PATCH] Switch test cases to use libcheck framework This keeps the test logic the same, but moves most of them into the libcheck test suite framework. It gives better control over grouping related tests, running them in parallel, and reporting on multiple failures. Running in parallel also brings problems, so libykcs11 tests are left untouched. Parallel access to a single hardware DUT does not make sense, and pcsc-lite doesn't work after a fork() in OS X 10.11+, so it can't run in libcheck's tests anyway. --- configure.ac | 1 + lib/tests/Makefile.am | 4 +- lib/tests/basic.c | 91 +++++++++++++++++++++++++--------------- lib/tests/parse_key.c | 61 +++++++++++++++++---------- tool/tests/Makefile.am | 4 +- tool/tests/parse_name.c | 66 +++++++++++++++++++++-------- tool/tests/test_inout.c | 51 ++++++++++++++++++---- ykcs11/tests/Makefile.am | 4 +- 8 files changed, 194 insertions(+), 88 deletions(-) diff --git a/configure.ac b/configure.ac index 2f5944c..7e5f5e4 100644 --- a/configure.ac +++ b/configure.ac @@ -50,6 +50,7 @@ AM_MISSING_PROG(GENGETOPT, gengetopt, $missing_dir) PKG_PROG_PKG_CONFIG PKG_CHECK_MODULES(OPENSSL, libcrypto) +PKG_CHECK_MODULES([CHECK], [check >= 0.9.6]) gl_LD_VERSION_SCRIPT gl_VALGRIND_TESTS diff --git a/lib/tests/Makefile.am b/lib/tests/Makefile.am index d1ae1b3..f83a46f 100644 --- a/lib/tests/Makefile.am +++ b/lib/tests/Makefile.am @@ -25,10 +25,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -AM_CFLAGS = $(WARN_CFLAGS) +AM_CFLAGS = $(WARN_CFLAGS) @CHECK_CFLAGS@ AM_CPPFLAGS = -I$(top_srcdir)/lib -I$(top_builddir)/lib -AM_LDFLAGS = -no-install +AM_LDFLAGS = -no-install @CHECK_LIBS@ LDADD = ../libykpiv.la check_PROGRAMS = basic parse_key diff --git a/lib/tests/basic.c b/lib/tests/basic.c index 85619ac..49b6f6b 100644 --- a/lib/tests/basic.c +++ b/lib/tests/basic.c @@ -35,41 +35,64 @@ #include #include +#include + +START_TEST(test_version_string) { + if (strcmp(YKPIV_VERSION_STRING, ykpiv_check_version(NULL)) != 0) { + ck_abort_msg("version mismatch %s != %s\n", YKPIV_VERSION_STRING, + ykpiv_check_version(NULL)); + } + + if (ykpiv_check_version(YKPIV_VERSION_STRING) == NULL) { + ck_abort_msg("version NULL?\n"); + } + + if (ykpiv_check_version("99.99.99") != NULL) { + ck_abort_msg("version not NULL?\n"); + } + + fprintf(stderr, "ykpiv version: header %s library %s\n", + YKPIV_VERSION_STRING, ykpiv_check_version (NULL)); +} +END_TEST + +START_TEST(test_strerror) { + const char *s; + + if (ykpiv_strerror(YKPIV_OK) == NULL) { + ck_abort_msg("ykpiv_strerror NULL\n"); + } + + s = ykpiv_strerror_name(YKPIV_OK); + if (s == NULL || strcmp(s, "YKPIV_OK") != 0) { + ck_abort_msg("ykpiv_strerror_name %s\n", s); + } +} +END_TEST + +Suite *basic_suite(void) { + Suite *s; + TCase *tc; + + s = suite_create("libykpiv basic"); + tc = tcase_create("basic"); + tcase_add_test(tc, test_version_string); + tcase_add_test(tc, test_strerror); + suite_add_tcase(s, tc); + + return s; +} + int main(void) { - if(strcmp(YKPIV_VERSION_STRING, ykpiv_check_version (NULL)) != 0) { - printf("version mismatch %s != %s\n", YKPIV_VERSION_STRING, - ykpiv_check_version(NULL)); - return EXIT_FAILURE; - } + int number_failed; + Suite *s; + SRunner *sr; - if(ykpiv_check_version(YKPIV_VERSION_STRING) == NULL) { - printf("version NULL?\n"); - return EXIT_FAILURE; - } - - if(ykpiv_check_version("99.99.99") != NULL) { - printf ("version not NULL?\n"); - return EXIT_FAILURE; - } - - printf ("ykpiv version: header %s library %s\n", - YKPIV_VERSION_STRING, ykpiv_check_version (NULL)); - - - if(ykpiv_strerror(YKPIV_OK) == NULL) { - printf ("ykpiv_strerror NULL\n"); - return EXIT_FAILURE; - } - - { - const char *s; - s = ykpiv_strerror_name(YKPIV_OK); - if(s == NULL || strcmp(s, "YKPIV_OK") != 0) { - printf("ykpiv_strerror_name %s\n", s); - return EXIT_FAILURE; - } - } - - return EXIT_SUCCESS; + s = basic_suite(); + sr = srunner_create(s); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/lib/tests/parse_key.c b/lib/tests/parse_key.c index 7740823..0a339c9 100644 --- a/lib/tests/parse_key.c +++ b/lib/tests/parse_key.c @@ -32,6 +32,8 @@ #include #include +#include + #include "ykpiv.h" struct key { @@ -57,30 +59,43 @@ static int parse_key(const char *text, const unsigned char *expected, int valid) unsigned char key[24]; size_t len = sizeof(key); ykpiv_rc res = ykpiv_hex_decode(text, strlen(text), key, &len); - if(res != YKPIV_OK && valid == 1) { - printf("key check failed for %s!\n", text); - return EXIT_FAILURE; - } else if(res != YKPIV_OK && valid == 0) { - return EXIT_SUCCESS; - } - - if(memcmp(expected, key, 24) != 0) { - printf("keys not matching for %s!\n", text); - return EXIT_FAILURE; - } - + if (valid) { + ck_assert(res == YKPIV_OK); + ck_assert(memcmp(expected, key, 24) == 0); + } else { + ck_assert(res != YKPIV_OK); + } return EXIT_SUCCESS; } -int main(void) { - size_t i; - - for(i = 0; i < sizeof(keys) / sizeof(struct key); i++) { - int res = parse_key(keys[i].text, keys[i].formatted, keys[i].valid); - if(res != EXIT_SUCCESS) { - return res; - } - } - - return EXIT_SUCCESS; +START_TEST(test_parse_key) { + int res = parse_key(keys[_i].text, keys[_i].formatted, keys[_i].valid); + ck_assert(res == EXIT_SUCCESS); +} +END_TEST + +Suite *parsekey_suite(void) { + Suite *s; + TCase *tc; + + s = suite_create("libykpiv parsekey"); + tc = tcase_create("parsekey"); + tcase_add_loop_test(tc, test_parse_key, 0, sizeof(keys) / sizeof(struct key)); + suite_add_tcase(s, tc); + + return s; +} + +int main(void) +{ + int number_failed; + Suite *s; + SRunner *sr; + + s = parsekey_suite(); + sr = srunner_create(s); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/tool/tests/Makefile.am b/tool/tests/Makefile.am index 64fd65f..1d76905 100644 --- a/tool/tests/Makefile.am +++ b/tool/tests/Makefile.am @@ -29,12 +29,12 @@ TESTS_ENVIRONMENT = export VERSION=$(PACKAGE_VERSION); export EXEEXT=$(EXEEXT); LOG_COMPILER = $(VALGRIND) -AM_CFLAGS = $(WARN_CFLAGS) +AM_CFLAGS = $(WARN_CFLAGS) @CHECK_CFLAGS@ AM_CPPFLAGS = -I$(top_srcdir)/lib -I$(top_builddir)/lib AM_CPPFLAGS += -I$(top_srcdir)/tool -I$(top_builddir)/tool AM_CPPFLAGS += $(OPENSSL_CFLAGS) -AM_LDFLAGS = -no-install +AM_LDFLAGS = -no-install @CHECK_LIBS@ parse_name_LDADD = ../libpiv_util.la $(OPENSSL_LIBS) test_inout_LDADD = ../libpiv_util.la diff --git a/tool/tests/parse_name.c b/tool/tests/parse_name.c index d6a103b..455a4e9 100644 --- a/tool/tests/parse_name.c +++ b/tool/tests/parse_name.c @@ -28,6 +28,8 @@ * */ +#include + #include #include @@ -39,18 +41,26 @@ #include "util.h" -static void test_name(const char *name, const char *expected, bool fail) { +struct name { + const char *name; + const char *parsed_name; + bool valid; +} names[] = { + {"/CN=test foo/", "CN = test foo", true}, + {"/CN=test/OU=bar/O=EXAMPLE/", "CN = test, OU = bar, O = EXAMPLE", true}, + {"/CN=test/OU=bar/O=EXAMPLE/", "CN = test, OU = wrong, O = EXAMPLE", false}, + {"/foo/", "", false}, + {"/CN=test/foobar/", "", false}, + {"/CN=test/foo=bar/", "", false}, +}; + +static bool test_name(const char *name, const char *expected) { char buf[1024]; BIO *bio; const char none[] = {0}; X509_NAME *parsed = parse_name(name); if(parsed == NULL) { - if(fail) { - return; - } else { - printf("Failed parsing of '%s'!\n", name); - exit(EXIT_FAILURE); - } + return false; } bio = BIO_new(BIO_s_mem()); @@ -61,17 +71,39 @@ static void test_name(const char *name, const char *expected, bool fail) { BIO_free(bio); X509_NAME_free(parsed); if(strcmp(buf, expected) != 0) { - printf("Names not matching: '%s' != '%s'\n", expected, buf); - exit(EXIT_FAILURE); + fprintf(stderr, "Names not matching: '%s' != '%s'\n", expected, buf); + return false; } + return true; } -int main(void) { - test_name("/CN=test foo/", "CN = test foo", false); - test_name("/CN=test/OU=bar/O=EXAMPLE/", "CN = test, OU = bar, O = EXAMPLE", false); - test_name("/foo/", "", true); - test_name("/CN=test/foobar/", "", true); - test_name("/CN=test/foo=bar/", "", true); - - return EXIT_SUCCESS; +START_TEST(test_parse_name) { + ck_assert(test_name(names[_i].name, names[_i].parsed_name) == names[_i].valid); +} +END_TEST + +Suite *test_suite(void) { + Suite *s; + TCase *tc; + + s = suite_create("yubico-piv-tool parse_name"); + tc = tcase_create("parse_name"); + tcase_add_loop_test(tc, test_parse_name, 0, sizeof(names) / sizeof(struct name)); + suite_add_tcase(s, tc); + + return s; +} + +int main(void) +{ + int number_failed; + Suite *s; + SRunner *sr; + + s = test_suite(); + sr = srunner_create(s); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/tool/tests/test_inout.c b/tool/tests/test_inout.c index fb475bf..a93c27c 100644 --- a/tool/tests/test_inout.c +++ b/tool/tests/test_inout.c @@ -28,6 +28,8 @@ * */ +#include + #include #include #include @@ -40,25 +42,58 @@ #define pipe(fds) _pipe(fds,4096, 0) #endif -static void test_inout(enum enum_format format) { +enum enum_format formats[] = { + format_arg_base64, + format_arg_hex, + format_arg_binary, +}; + +static bool inout(enum enum_format format) { const unsigned char buf[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; unsigned char buf2[sizeof(buf)]; int pipefd[2]; FILE *tmp1, *tmp2; - assert(pipe(pipefd) == 0); + if (pipe(pipefd) != 0) + return false; tmp1 = fdopen(pipefd[1], "w"); dump_data(buf, sizeof(buf), tmp1, false, format); fclose(tmp1); tmp2 = fdopen(pipefd[0], "r"); read_data(buf2, sizeof(buf2), tmp2, format); - assert(memcmp(buf, buf2, sizeof(buf)) == 0); + if (memcmp(buf, buf2, sizeof(buf)) != 0) + return false; fclose(tmp2); + return true; } -int main(void) { - test_inout(format_arg_base64); - test_inout(format_arg_hex); - test_inout(format_arg_binary); - exit(0); +START_TEST(test_inout) { + ck_assert(inout(formats[_i])); +} +END_TEST + +Suite *test_suite(void) { + Suite *s; + TCase *tc; + + s = suite_create("yubico-piv-tool inout"); + tc = tcase_create("inout"); + tcase_add_loop_test(tc, test_inout, 0, sizeof(formats) / sizeof(*formats)); + suite_add_tcase(s, tc); + + return s; +} + +int main(void) +{ + int number_failed; + Suite *s; + SRunner *sr; + + s = test_suite(); + sr = srunner_create(s); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/ykcs11/tests/Makefile.am b/ykcs11/tests/Makefile.am index 1ee6545..3289626 100644 --- a/ykcs11/tests/Makefile.am +++ b/ykcs11/tests/Makefile.am @@ -29,12 +29,12 @@ #LOG_COMPILER = $(VALGRIND) -AM_CFLAGS = $(WARN_CFLAGS) +AM_CFLAGS = $(WARN_CFLAGS) @CHECK_CFLAGS@ AM_CPPFLAGS = -I$(top_srcdir)/lib -I$(top_builddir)/lib AM_CPPFLAGS += -I$(top_srcdir)/ykcs11 -I$(top_builddir)/ykcs11 AM_CPPFLAGS += $(OPENSSL_CFLAGS) -AM_LDFLAGS = -no-install +AM_LDFLAGS = -no-install @CHECK_LIBS@ ykcs11_tests_LDADD = ../libykcs11.la $(OPENSSL_LIBS)