Strip DER encoding from EC signatures.

ykcs11/utils.c

@@ -173,5 +173,25 @@ CK_BBOOL is_valid_key_id(CK_BYTE id) {
     return CK_FALSE;
 
   return CK_TRUE;
-  
+}
+
+void strip_DER_encoding_from_ECSIG(CK_BYTE_PTR data, CK_ULONG_PTR len) {
+
+  CK_BYTE_PTR  ptr;
+  CK_ULONG     n_len;
+
+  // Maximum DER length for P256 is 2 + 2 + 33 + 2 + 33 = 72
+  if (*len <= 72)
+    n_len = 32;
+  else
+    n_len = 48;
+
+  ptr = data + 4;
+  if (*ptr == 0)
+    ptr++;
+
+  memmove(data, ptr, n_len);
+  memmove(data+n_len, data + *len - n_len, n_len);
+
+  *len = n_len * 2;
 }

ykcs11/utils.h

@@ -11,4 +11,6 @@ void  destroy_token(ykcs11_slot_t *slot);
 
 CK_BBOOL is_valid_key_id(CK_BYTE id);
 
+void strip_DER_encoding_from_ECSIG(CK_BYTE_PTR data, CK_ULONG_PTR len);
+
 #endif

ykcs11/ykcs11.c

@@ -1490,6 +1490,15 @@ CK_DEFINE_FUNCTION(CK_RV, C_Sign)(
   DBG(("Got %lu bytes back", *pulSignatureLen));
   dump_hex(pSignature, *pulSignatureLen, stderr, CK_TRUE);
 
+  if (!is_RSA_mechanism(op_info.mechanism.mechanism)) {
+    // ECDSA, we must remove the DER encoding and only return R,S
+    // as required by the specs
+    strip_DER_encoding_from_ECSIG(pSignature, pulSignatureLen);
+
+    DBG(("After removing DER encoding %lu", *pulSignatureLen));
+    dump_hex(pSignature, *pulSignatureLen, stderr, CK_TRUE);
+  }
+
   op_info.type = YKCS11_NOOP; // TODO: anything to clear here?
 
   DOUT;