Switch test cases to use libcheck framework

This keeps the test logic the same, but moves most of them into the libcheck
test suite framework.  It gives better control over grouping related tests,
running them in parallel, and reporting on multiple failures.

Running in parallel also brings problems, so libykcs11 tests are left
untouched.  Parallel access to a single hardware DUT does not make sense,
and pcsc-lite doesn't work after a fork() in OS X 10.11+, so it can't run
in libcheck's tests anyway.

configure.ac

@@ -50,6 +50,7 @@ AM_MISSING_PROG(GENGETOPT, gengetopt, $missing_dir)
 PKG_PROG_PKG_CONFIG
 
 PKG_CHECK_MODULES(OPENSSL, libcrypto)
+PKG_CHECK_MODULES([CHECK], [check >= 0.9.6])
 
 gl_LD_VERSION_SCRIPT
 gl_VALGRIND_TESTS

lib/tests/Makefile.am

@@ -25,10 +25,10 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-AM_CFLAGS = $(WARN_CFLAGS)
+AM_CFLAGS = $(WARN_CFLAGS) @CHECK_CFLAGS@
 AM_CPPFLAGS = -I$(top_srcdir)/lib -I$(top_builddir)/lib
 
-AM_LDFLAGS = -no-install
+AM_LDFLAGS = -no-install @CHECK_LIBS@
 LDADD = ../libykpiv.la
 
 check_PROGRAMS = basic parse_key

lib/tests/basic.c

@@ -35,41 +35,64 @@
 #include <stdlib.h>
 #include <string.h>
 
-int main(void)
+#include <check.h>
-{
+
+START_TEST(test_version_string) {
   if (strcmp(YKPIV_VERSION_STRING, ykpiv_check_version(NULL)) != 0) {
-    printf("version mismatch %s != %s\n", YKPIV_VERSION_STRING,
+    ck_abort_msg("version mismatch %s != %s\n", YKPIV_VERSION_STRING,
                  ykpiv_check_version(NULL));
-    return EXIT_FAILURE;
   }
 
   if (ykpiv_check_version(YKPIV_VERSION_STRING) == NULL) {
-    printf("version NULL?\n");
+    ck_abort_msg("version NULL?\n");
-    return EXIT_FAILURE;
   }
 
   if (ykpiv_check_version("99.99.99") != NULL) {
-    printf ("version not NULL?\n");
+    ck_abort_msg("version not NULL?\n");
-    return EXIT_FAILURE;
   }
 
-  printf ("ykpiv version: header %s library %s\n",
+  fprintf(stderr, "ykpiv version: header %s library %s\n",
           YKPIV_VERSION_STRING, ykpiv_check_version (NULL));
+}
+END_TEST
 
+START_TEST(test_strerror) {
+  const char *s;
 
   if (ykpiv_strerror(YKPIV_OK) == NULL) {
-    printf ("ykpiv_strerror NULL\n");
+    ck_abort_msg("ykpiv_strerror NULL\n");
-    return EXIT_FAILURE;
   }
 
-  {
-    const char *s;
   s = ykpiv_strerror_name(YKPIV_OK);
   if (s == NULL || strcmp(s, "YKPIV_OK") != 0) {
-      printf("ykpiv_strerror_name %s\n", s);
+    ck_abort_msg("ykpiv_strerror_name %s\n", s);
-      return EXIT_FAILURE;
   }
 }
+END_TEST
+
+Suite *basic_suite(void) {
+  Suite *s;
+  TCase *tc;
+
+  s = suite_create("libykpiv basic");
+  tc = tcase_create("basic");
+  tcase_add_test(tc, test_version_string);
+  tcase_add_test(tc, test_strerror);
+  suite_add_tcase(s, tc);
+
+  return s;
+}
 
-  return EXIT_SUCCESS;
+int main(void)
+{
+  int number_failed;
+  Suite *s;
+  SRunner *sr;
+
+  s = basic_suite();
+  sr = srunner_create(s);
+  srunner_run_all(sr, CK_NORMAL);
+  number_failed = srunner_ntests_failed(sr);
+  srunner_free(sr);
+  return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
 }

lib/tests/parse_key.c

@@ -32,6 +32,8 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <check.h>
+
 #include "ykpiv.h"
 
 struct key {
@@ -57,30 +59,43 @@ static int parse_key(const char *text, const unsigned char *expected, int valid)
 	unsigned char key[24];
 	size_t len = sizeof(key);
 	ykpiv_rc res = ykpiv_hex_decode(text, strlen(text), key, &len);
-	if(res != YKPIV_OK && valid == 1) {
+  if (valid) {
-		printf("key check failed for %s!\n", text);
+    ck_assert(res == YKPIV_OK);
-		return EXIT_FAILURE;
+    ck_assert(memcmp(expected, key, 24) == 0);
-	} else if(res != YKPIV_OK && valid == 0) {
+  } else {
+    ck_assert(res != YKPIV_OK);
+  }
 	return EXIT_SUCCESS;
 }
 
-	if(memcmp(expected, key, 24) != 0) {
+START_TEST(test_parse_key) {
-		printf("keys not matching for %s!\n", text);
+  int res = parse_key(keys[_i].text, keys[_i].formatted, keys[_i].valid);
-		return EXIT_FAILURE;
+  ck_assert(res == EXIT_SUCCESS);
+}
+END_TEST
+
+Suite *parsekey_suite(void) {
+  Suite *s;
+  TCase *tc;
+
+  s = suite_create("libykpiv parsekey");
+  tc = tcase_create("parsekey");
+  tcase_add_loop_test(tc, test_parse_key, 0, sizeof(keys) / sizeof(struct key));
+  suite_add_tcase(s, tc);
+
+  return s;
 }
 
-	return EXIT_SUCCESS;
+int main(void)
-}
+{
+  int number_failed;
+  Suite *s;
+  SRunner *sr;
 
-int main(void) {
+  s = parsekey_suite();
-	size_t i;
+  sr = srunner_create(s);
-
+  srunner_run_all(sr, CK_NORMAL);
-	for(i = 0; i < sizeof(keys) / sizeof(struct key); i++) {
+  number_failed = srunner_ntests_failed(sr);
-		int res = parse_key(keys[i].text, keys[i].formatted, keys[i].valid);
+  srunner_free(sr);
-		if(res != EXIT_SUCCESS) {
+  return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
-			return res;
-		}
-	}
-
-	return EXIT_SUCCESS;
 }

tool/tests/Makefile.am

@@ -29,12 +29,12 @@ TESTS_ENVIRONMENT = export VERSION=$(PACKAGE_VERSION); export EXEEXT=$(EXEEXT);
 
 LOG_COMPILER = $(VALGRIND)
 
-AM_CFLAGS = $(WARN_CFLAGS)
+AM_CFLAGS = $(WARN_CFLAGS) @CHECK_CFLAGS@
 AM_CPPFLAGS = -I$(top_srcdir)/lib -I$(top_builddir)/lib
 AM_CPPFLAGS += -I$(top_srcdir)/tool -I$(top_builddir)/tool
 AM_CPPFLAGS += $(OPENSSL_CFLAGS)
 
-AM_LDFLAGS = -no-install
+AM_LDFLAGS = -no-install @CHECK_LIBS@
 
 parse_name_LDADD = ../libpiv_util.la $(OPENSSL_LIBS)
 test_inout_LDADD = ../libpiv_util.la

tool/tests/parse_name.c

@@ -28,6 +28,8 @@
  *
  */
 
+#include <check.h>
+
 #include <string.h>
 #include <stdbool.h>
 
@@ -39,18 +41,26 @@
 
 #include "util.h"
 
-static void test_name(const char *name, const char *expected, bool fail) {
+struct name {
+  const char *name;
+  const char *parsed_name;
+  bool valid;
+} names[] = {
+  {"/CN=test foo/", "CN = test foo", true},
+  {"/CN=test/OU=bar/O=EXAMPLE/", "CN = test, OU = bar, O = EXAMPLE", true},
+  {"/CN=test/OU=bar/O=EXAMPLE/", "CN = test, OU = wrong, O = EXAMPLE", false},
+  {"/foo/", "", false},
+  {"/CN=test/foobar/", "", false},
+  {"/CN=test/foo=bar/", "", false},
+};
+
+static bool test_name(const char *name, const char *expected) {
   char buf[1024];
   BIO *bio;
   const char none[] = {0};
   X509_NAME *parsed = parse_name(name);
   if(parsed == NULL) {
-    if(fail) {
+    return false;
-      return;
-    } else {
-      printf("Failed parsing of '%s'!\n", name);
-      exit(EXIT_FAILURE);
-    }
   }
 
   bio = BIO_new(BIO_s_mem());
@@ -61,17 +71,39 @@ static void test_name(const char *name, const char *expected, bool fail) {
   BIO_free(bio);
   X509_NAME_free(parsed);
   if(strcmp(buf, expected) != 0) {
-    printf("Names not matching: '%s' != '%s'\n", expected, buf);
+    fprintf(stderr, "Names not matching: '%s' != '%s'\n", expected, buf);
-    exit(EXIT_FAILURE);
+    return false;
   }
+  return true;
 }
 
-int main(void) {
+START_TEST(test_parse_name) {
-  test_name("/CN=test foo/", "CN = test foo", false);
+  ck_assert(test_name(names[_i].name, names[_i].parsed_name) == names[_i].valid);
-  test_name("/CN=test/OU=bar/O=EXAMPLE/", "CN = test, OU = bar, O = EXAMPLE", false);
+}
-  test_name("/foo/", "", true);
+END_TEST
-  test_name("/CN=test/foobar/", "", true);
+
-  test_name("/CN=test/foo=bar/", "", true);
+Suite *test_suite(void) {
-
+  Suite *s;
-  return EXIT_SUCCESS;
+  TCase *tc;
+
+  s = suite_create("yubico-piv-tool parse_name");
+  tc = tcase_create("parse_name");
+  tcase_add_loop_test(tc, test_parse_name, 0, sizeof(names) / sizeof(struct name));
+  suite_add_tcase(s, tc);
+
+  return s;
+}
+
+int main(void)
+{
+  int number_failed;
+  Suite *s;
+  SRunner *sr;
+
+  s = test_suite();
+  sr = srunner_create(s);
+  srunner_run_all(sr, CK_NORMAL);
+  number_failed = srunner_ntests_failed(sr);
+  srunner_free(sr);
+  return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
 }

tool/tests/test_inout.c

@@ -28,6 +28,8 @@
  *
  */
 
+#include <check.h>
+
 #include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
@@ -40,25 +42,58 @@
 #define pipe(fds) _pipe(fds,4096, 0)
 #endif
 
-static void test_inout(enum enum_format format) {
+enum enum_format formats[] = {
+  format_arg_base64,
+  format_arg_hex,
+  format_arg_binary,
+};
+
+static bool inout(enum enum_format format) {
   const unsigned char buf[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
   unsigned char buf2[sizeof(buf)];
   int pipefd[2];
   FILE *tmp1, *tmp2;
 
-  assert(pipe(pipefd) == 0);
+  if (pipe(pipefd) != 0)
+    return false;
   tmp1 = fdopen(pipefd[1], "w");
   dump_data(buf, sizeof(buf), tmp1, false, format);
   fclose(tmp1);
   tmp2 = fdopen(pipefd[0], "r");
   read_data(buf2, sizeof(buf2), tmp2, format);
-  assert(memcmp(buf, buf2, sizeof(buf)) == 0);
+  if (memcmp(buf, buf2, sizeof(buf)) != 0)
+    return false;
   fclose(tmp2);
+  return true;
 }
 
-int main(void) {
+START_TEST(test_inout) {
-  test_inout(format_arg_base64);
+  ck_assert(inout(formats[_i]));
-  test_inout(format_arg_hex);
+}
-  test_inout(format_arg_binary);
+END_TEST
-  exit(0);
+
+Suite *test_suite(void) {
+  Suite *s;
+  TCase *tc;
+
+  s = suite_create("yubico-piv-tool inout");
+  tc = tcase_create("inout");
+  tcase_add_loop_test(tc, test_inout, 0, sizeof(formats) / sizeof(*formats));
+  suite_add_tcase(s, tc);
+
+  return s;
+}
+
+int main(void)
+{
+  int number_failed;
+  Suite *s;
+  SRunner *sr;
+
+  s = test_suite();
+  sr = srunner_create(s);
+  srunner_run_all(sr, CK_NORMAL);
+  number_failed = srunner_ntests_failed(sr);
+  srunner_free(sr);
+  return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
 }

ykcs11/tests/Makefile.am

@@ -29,12 +29,12 @@
 
 #LOG_COMPILER = $(VALGRIND)
 
-AM_CFLAGS = $(WARN_CFLAGS)
+AM_CFLAGS = $(WARN_CFLAGS) @CHECK_CFLAGS@
 AM_CPPFLAGS = -I$(top_srcdir)/lib -I$(top_builddir)/lib
 AM_CPPFLAGS += -I$(top_srcdir)/ykcs11 -I$(top_builddir)/ykcs11
 AM_CPPFLAGS += $(OPENSSL_CFLAGS)
 
-AM_LDFLAGS = -no-install
+AM_LDFLAGS = -no-install @CHECK_LIBS@
 
 ykcs11_tests_LDADD = ../libykcs11.la $(OPENSSL_LIBS)