fix PSS signing support

ykcs11/mechanisms.c

@@ -262,7 +262,6 @@ CK_RV apply_sign_mechanism_finalize(op_info_t *op_info) {
 
   CK_RV    rv;
   int      nid = NID_undef;
-  RSA      *rsa;
   CK_ULONG len;
 
   if (op_info->type != YKCS11_SIGN)
@@ -282,14 +281,8 @@ CK_RV apply_sign_mechanism_finalize(op_info_t *op_info) {
   case CKM_RSA_PKCS_PSS:
     // Compute padding for all PSS variants
     // TODO: digestinfo/paraminfo ?
-
+    rv = do_pkcs_pss(op_info->op.sign.key, op_info->buf, op_info->buf_len, nid, op_info->buf, &op_info->buf_len);
-    rv = do_encode_rsa_public_key(op_info->op.sign.key, op_info->op.sign.key_len, &rsa);
+    RSA_free(op_info->op.sign.key);
-    if (rv != CKR_OK)
-      return CKR_FUNCTION_FAILED;
-
-    rv = do_pkcs_pss(rsa, op_info->buf, op_info->buf_len, nid, op_info->buf, &op_info->buf_len);
-
-    // TODO: does rsa have to be free'd ?
 
     return rv;
 

ykcs11/openssl_utils.c

@@ -557,16 +557,13 @@ CK_RV do_pkcs_1_digest_info(CK_BYTE_PTR in, CK_ULONG in_len, int nid, CK_BYTE_PT
 
 CK_RV do_pkcs_pss(RSA *key, CK_BYTE_PTR in, CK_ULONG in_len, int nid,
           CK_BYTE_PTR out, CK_ULONG_PTR out_len) {
-  unsigned char em[512]; // Max for this is ceil((|key_len_bits| - 1) / 8)
+  unsigned char em[RSA_size(key)];
 
   OpenSSL_add_all_digests();
 
+  DBG("Apply PSS padding to %lu bytes and get %d", in_len, RSA_size(key));
+
   // TODO: rand must be seeded first (should be automatic)
-  if (*out_len < (CK_ULONG)RSA_size(key))
-    return CKR_BUFFER_TOO_SMALL;
-
-  DBG("Apply PSS padding to %lu bytes and get %d\n", in_len, RSA_size(key));
-
   if (out != in)
     memcpy(out, in, in_len);
 
@@ -576,7 +573,8 @@ CK_RV do_pkcs_pss(RSA *key, CK_BYTE_PTR in, CK_ULONG in_len, int nid,
     return CKR_FUNCTION_FAILED;
   }
 
-  *out_len = (CK_ULONG) RSA_size(key);
+  memcpy(out, em, sizeof(em));
+  *out_len = (CK_ULONG) sizeof(em);
 
   EVP_cleanup();
 

ykcs11/ykcs11.c

@@ -1687,11 +1687,13 @@ CK_DEFINE_FUNCTION(CK_RV, C_SignInit)(
 {
   CK_KEY_TYPE  type = 0;
   CK_ULONG     key_len = 0;
+  CK_BYTE      exp[3];
   CK_BYTE      buf[1024];
   CK_ATTRIBUTE template[] = {
     {CKA_KEY_TYPE, &type, sizeof(type)},
     {CKA_MODULUS_BITS, &key_len, sizeof(key_len)},
     {CKA_MODULUS, NULL, 0},
+    {CKA_PUBLIC_EXPONENT, exp, sizeof(exp)},
     {CKA_EC_POINT, buf, sizeof(buf)},
   };
 
@@ -1754,18 +1756,32 @@ CK_DEFINE_FUNCTION(CK_RV, C_SignInit)(
 
     // Also store the raw public key if the mechanism is PSS
     if (is_PSS_mechanism(pMechanism->mechanism)) {
-      op_info.op.sign.key = malloc(key_len);
+      op_info.op.sign.key = RSA_new();
       if (op_info.op.sign.key == NULL)
         return CKR_HOST_MEMORY;
 
-      template[2].pValue = op_info.op.sign.key;
+      template[2].pValue = buf;
-      template[2].ulValueLen = key_len;
+      template[2].ulValueLen = (key_len + 7) / 8 ;
 
       if (get_attribute(&session, hKey, template + 2) != CKR_OK) {
         DBG("Unable to get public key");
         return CKR_KEY_HANDLE_INVALID;
       }
+      op_info.op.sign.key->n = BN_bin2bn(buf, (key_len + 7) / 8, NULL);
+      if(op_info.op.sign.key->n == NULL) {
+        DBG("Failed to parse public key modulus.");
+        return CKR_KEY_HANDLE_INVALID;
+      }
 
+      if (get_attribute(&session, hKey, template + 3) != CKR_OK) {
+        DBG("Unable to get public exponent");
+        return CKR_KEY_HANDLE_INVALID;
+      }
+      op_info.op.sign.key->e = BN_bin2bn(exp, sizeof(exp), NULL);
+      if(op_info.op.sign.key->e == NULL) {
+        DBG("Failed to parse public key exponent.");
+        return CKR_KEY_HANDLE_INVALID;
+      }
     }
     else {
       op_info.op.sign.key = NULL;
@@ -1774,7 +1790,7 @@ CK_DEFINE_FUNCTION(CK_RV, C_SignInit)(
   }
   else {
     // ECDSA key
-    if (get_attribute(&session, hKey, template + 3) != CKR_OK) {
+    if (get_attribute(&session, hKey, template + 4) != CKR_OK) {
       DBG("Unable to get key length");
       return CKR_KEY_HANDLE_INVALID;
     }

ykcs11/ykcs11.h

@@ -75,7 +75,7 @@ typedef struct {
 
 typedef struct {
   ykcs11_md_ctx_t   *md_ctx; // Digest context
-  CK_BYTE_PTR       key;     // Raw public key (needed for PSS)
+  RSA               *key;    // Raw public key (needed for PSS)
   CK_BYTE           algo;    // Algo for ykpiv // TODO: infer this from the key length?
   CK_ULONG          key_id;  // Key id for ykpiv // TODO: make this a BYTE and store the id {0, 1, 2, 3}
   CK_ULONG          key_len; // Length in bits