diff --git a/tests/Makefile.am b/tests/Makefile.am
index a96a1bc..6480298 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -26,4 +26,4 @@
 
 TESTS = basic.sh
 
-AM_TESTS_ENVIRONMENT = export VERSION=$(VERSION);
+AM_TESTS_ENVIRONMENT = export VERSION=$(VERSION); export EXEEXT=$(EXEEXT);
diff --git a/tests/basic.sh b/tests/basic.sh
index 6ed19dd..f29f4c1 100755
--- a/tests/basic.sh
+++ b/tests/basic.sh
@@ -31,11 +31,11 @@
 
 set -e
 
-HELP_OUTPUT=$(../yubico-piv-tool --help)
+HELP_OUTPUT=$(../yubico-piv-tool$EXEEXT --help)
 
 expected="yubico-piv-tool $VERSION"
-VERSION_OUTPUT=$(../yubico-piv-tool --version)
+VERSION_OUTPUT=$(../yubico-piv-tool$EXEEXT --version | sed 's/\r//')
 if [ "x$VERSION_OUTPUT" != "x$expected" ]; then
-  echo "Version not matching expected output $expected."
+  echo "Version ($VERSION_OUTPUT) not matching expected output $expected."
   exit 1
 fi
diff --git a/windows.mk b/windows.mk
index 8fb94c1..434275a 100644
--- a/windows.mk
+++ b/windows.mk
@@ -61,7 +61,7 @@ doit:
 	zip -r ../../$(PACKAGE)-$(VERSION)-win$(ARCH).zip *
 
 32bit:
-	$(MAKE) -f windows.mk doit ARCH=32 HOST=i686-w64-mingw32
+	$(MAKE) -f windows.mk doit ARCH=32 HOST=i686-w64-mingw32 CHECK=CHECK
 
 64bit:
-	$(MAKE) -f windows.mk doit ARCH=64 HOST=x86_64-w64-mingw32
+	$(MAKE) -f windows.mk doit ARCH=64 HOST=x86_64-w64-mingw32 CHECK=CHECK