rsa: Adds a signer for PSS and PKCS1#1.5

This adds a mockhsm implementation and provides an integration test for
both signers.

Cargo.toml

@@ -22,6 +22,7 @@ bitflags = "2"
 cmac = "0.7"
 cbc = "0.1"
 ccm = { version = "0.5", features = ["std"] }
+digest = { version = "0.10", default-features = false }
 ecdsa = { version = "0.16", default-features = false, features = ["pkcs8"] }
 ed25519 = "2"
 log = "0.4"
@@ -32,6 +33,7 @@ serde = { version = "1", features = ["serde_derive"] }
 rand_core = { version = "0.6", features = ["std"] }
 rsa = "0.9.6"
 signature = { version = "2", features = ["derive"] }
+sha1 = { version = "0.10", features = ["oid"] }
 sha2 = { version = "0.10", features = ["oid"] }
 spki = { version = "0.7.3", default-features = false }
 subtle = "2"
@@ -41,7 +43,6 @@ uuid = { version = "1", default-features = false }
 zeroize = { version = "1", features = ["zeroize_derive"] }
 
 # optional dependencies
-digest = { version = "0.10", optional = true, default-features = false }
 ed25519-dalek = { version = "2", optional = true, features = ["rand_core"] }
 hmac = { version = "0.12", optional = true }
 k256 = { version = "0.13", optional = true, features = ["ecdsa", "sha256"] }
@@ -53,14 +54,15 @@ tiny_http = { version = "0.12", optional = true }
 [dev-dependencies]
 ed25519-dalek = "2"
 once_cell = "1"
+rsa = { version = "0.9.6", features = ["sha1", "sha2"] }
 p256 = { version = "0.13", features = ["ecdsa"] }
 x509-cert = { version = "0.2.4", features = ["builder"] }
 
 [features]
 default = ["http", "passwords", "setup"]
 http-server = ["tiny_http"]
 http = []
-mockhsm = ["digest", "ecdsa/arithmetic", "ed25519-dalek", "p256/ecdsa", "secp256k1"]
+mockhsm = ["ecdsa/arithmetic", "ed25519-dalek", "p256/ecdsa", "secp256k1"]
 passwords = ["hmac", "pbkdf2"]
 secp256k1 = ["k256"]
 setup = ["passwords", "serde_json", "uuid/serde"]

src/client.rs

@@ -28,13 +28,14 @@ use crate::{
     object::{self, commands::*, generate},
     opaque::{self, commands::*},
     otp::{self, commands::*},
-    rsa::{self, oaep::commands::*},
+    rsa::{self, oaep::commands::*, pkcs1::commands::*, pss::commands::*, SignatureAlgorithm},
     serialization::{deserialize, serialize},
     session::{self, Session},
     template::{commands::*, Template},
     uuid,
     wrap::{self, commands::*},
 };
+use sha2::Sha256;
 use std::{
     sync::{Arc, Mutex},
     time::{Duration, Instant},
@@ -44,14 +45,10 @@ use std::{
 use std::{thread, time::SystemTime};
 
 #[cfg(feature = "untested")]
-use {
-    crate::{
-        algorithm::Algorithm,
-        ecdh::{self, commands::*},
-        rsa::{pkcs1::commands::*, pss::commands::*},
-        ssh::{self, commands::*},
-    },
-    sha2::{Digest, Sha256},
+use crate::{
+    algorithm::Algorithm,
+    ecdh::{self, commands::*},
+    ssh::{self, commands::*},
 };
 
 #[cfg(any(doc, docsrs))]
@@ -1019,64 +1016,71 @@ impl Client {
 
     /// Compute an RSASSA-PKCS#1v1.5 signature of the SHA-256 hash of the given data.
     ///
-    /// **WARNING**: This functionality has not been tested and has not yet been
-    /// confirmed to actually work! USE AT YOUR OWN RISK!
-    ///
-    /// You will need to enable the `untested` cargo feature to use it.
-    ///
     /// <https://developers.yubico.com/YubiHSM2/Commands/Sign_Pkcs1.html>
-    #[cfg(feature = "untested")]
-    pub fn sign_rsa_pkcs1v15_sha256(
+    pub(crate) fn sign_rsa_pkcs1v15<S: SignatureAlgorithm>(
         &self,
         key_id: object::Id,
         data: &[u8],
     ) -> Result<rsa::pkcs1::Signature, Error> {
         Ok(self
             .send_command(SignPkcs1Command {
                 key_id,
-                digest: Sha256::digest(data).as_slice().into(),
+                digest: S::digest(data).as_slice().into(),
             })?
             .into())
     }
 
-    /// Compute an RSASSA-PSS signature of the SHA-256 hash of the given data with the given key ID.
-    ///
-    /// **WARNING**: This functionality has not been tested and has not yet been
-    /// confirmed to actually work! USE AT YOUR OWN RISK!
+    /// Compute an RSASSA-PKCS#1v1.5 signature of the SHA-256 hash of the given data.
     ///
-    /// You will need to enable the `untested` cargo feature to use it.
+    /// <https://developers.yubico.com/YubiHSM2/Commands/Sign_Pkcs1.html>
+    pub fn sign_rsa_pkcs1v15_sha256(
+        &self,
+        key_id: object::Id,
+        data: &[u8],
+    ) -> Result<rsa::pkcs1::Signature, Error> {
+        self.sign_rsa_pkcs1v15::<Sha256>(key_id, data)
+    }
+
+    /// Compute an RSASSA-PSS signature of the SHA-256 hash of the given data with the given key ID.
     ///
     /// <https://developers.yubico.com/YubiHSM2/Commands/Sign_Pss.html>
-    #[cfg(feature = "untested")]
-    pub fn sign_rsa_pss_sha256(
+    pub(crate) fn sign_rsa_pss<S: SignatureAlgorithm>(
         &self,
         key_id: object::Id,
         data: &[u8],
     ) -> Result<rsa::pss::Signature, Error> {
         ensure!(
-            data.len() > rsa::pss::MAX_MESSAGE_SIZE,
+            data.len() < rsa::pss::MAX_MESSAGE_SIZE,
             ErrorKind::ProtocolError,
             "message too large to be signed (max: {})",
             rsa::pss::MAX_MESSAGE_SIZE
         );
 
-        let mut hasher = Sha256::default();
-
-        let length = data.len() as u16;
-        hasher.update(length.to_be_bytes());
+        let mut hasher = S::new();
         hasher.update(data);
         let digest = hasher.finalize();
 
         Ok(self
             .send_command(SignPssCommand {
                 key_id,
-                mgf1_hash_alg: rsa::mgf::Algorithm::Sha256,
+                mgf1_hash_alg: S::MGF_ALGORITHM,
                 salt_len: digest.as_slice().len() as u16,
                 digest: digest.as_slice().into(),
             })?
             .into())
     }
 
+    /// Compute an RSASSA-PSS signature of the SHA-256 hash of the given data with the given key ID.
+    ///
+    /// <https://developers.yubico.com/YubiHSM2/Commands/Sign_Pss.html>
+    pub fn sign_rsa_pss_sha256(
+        &self,
+        key_id: object::Id,
+        data: &[u8],
+    ) -> Result<rsa::pss::Signature, Error> {
+        self.sign_rsa_pss::<Sha256>(key_id, data)
+    }
+
     /// Sign an SSH certificate using the given template.
     ///
     /// **WARNING**: This functionality has not been tested and has not yet been

src/mockhsm/command.rs

@@ -17,7 +17,7 @@ use crate::{
     opaque::{self, commands::*},
     otp,
     response::{self, Response},
-    rsa,
+    rsa::{self, pkcs1::commands::*, pss::commands::*},
     serialization::deserialize,
     session::{self, commands::*},
     template,
@@ -29,9 +29,18 @@ use ::ecdsa::{
     hazmat::SignPrimitive,
 };
 use ::hmac::{Hmac, Mac};
+use ::rsa::{pkcs1v15, pss, RsaPrivateKey};
+use digest::{
+    const_oid::AssociatedOid, crypto_common::OutputSizeUser, typenum::Unsigned, Digest,
+    FixedOutputReset,
+};
 use rand_core::{OsRng, RngCore};
-use sha2::Sha256;
-use signature::Signer;
+use sha1::Sha1;
+use sha2::{Sha256, Sha384, Sha512};
+use signature::{
+    hazmat::{PrehashSigner, RandomizedPrehashSigner},
+    Signer,
+};
 use std::{io::Cursor, str::FromStr};
 use subtle::ConstantTimeEq;
 
@@ -119,6 +128,8 @@ pub(crate) fn session_message(
         Code::SignEddsa => sign_eddsa(state, &command.data),
         Code::GetStorageInfo => get_storage_info(),
         Code::VerifyHmac => verify_hmac(state, &command.data),
+        Code::SignPss => sign_pss(state, &command.data),
+        Code::SignPkcs1 => sign_pkcs1v15(state, &command.data),
         unsupported => panic!("unsupported command type: {unsupported:?}"),
     };
 
@@ -707,6 +718,104 @@ fn sign_hmac(state: &State, cmd_data: &[u8]) -> response::Message {
     }
 }
 
+/// Sign a message using the RSASSA-PSS signature algorithm
+fn sign_pss(state: &State, cmd_data: &[u8]) -> response::Message {
+    #[inline]
+    fn sign_pss_digest<D: Digest + FixedOutputReset>(
+        private_key: &RsaPrivateKey,
+        msg: &[u8],
+    ) -> pss::Signature {
+        let signing_key = pss::SigningKey::<D>::new(private_key.clone());
+        signing_key
+            .sign_prehash_with_rng(&mut OsRng, msg)
+            .expect("unable to sign with prehash, wrong payload length?")
+    }
+
+    let command: SignPssCommand =
+        deserialize(cmd_data).unwrap_or_else(|e| panic!("error parsing Code::SignPss: {e:?}"));
+
+    if let Some(obj) = state
+        .objects
+        .get(command.key_id, object::Type::AsymmetricKey)
+    {
+        if let Payload::RsaKey(private_key) = &obj.payload {
+            let signature = match command.mgf1_hash_alg {
+                rsa::mgf::Algorithm::Sha1 => {
+                    sign_pss_digest::<Sha1>(private_key, command.digest.as_ref())
+                }
+                rsa::mgf::Algorithm::Sha256 => {
+                    sign_pss_digest::<Sha256>(private_key, command.digest.as_ref())
+                }
+                rsa::mgf::Algorithm::Sha384 => {
+                    sign_pss_digest::<Sha384>(private_key, command.digest.as_ref())
+                }
+                rsa::mgf::Algorithm::Sha512 => {
+                    sign_pss_digest::<Sha512>(private_key, command.digest.as_ref())
+                }
+            };
+
+            SignPssResponse((&signature).into()).serialize()
+        } else {
+            debug!("not an Rsa key: {:?}", obj.algorithm());
+            device::ErrorKind::InvalidCommand.into()
+        }
+    } else {
+        debug!("no such object ID: {:?}", command.key_id);
+        device::ErrorKind::ObjectNotFound.into()
+    }
+}
+
+/// Sign a message using the RSASSA-PKCS1-v1_5 signature algorithm
+fn sign_pkcs1v15(state: &State, cmd_data: &[u8]) -> response::Message {
+    #[inline]
+    fn sign_pkcs1v15_prehash<D: Digest + AssociatedOid>(
+        private_key: &RsaPrivateKey,
+        prehash: &[u8],
+    ) -> pkcs1v15::Signature {
+        let signing_key = pkcs1v15::SigningKey::<D>::new(private_key.clone());
+        signing_key
+            .sign_prehash(prehash)
+            .expect("unable to sign with prehash, wrong payload length?")
+    }
+
+    let command: SignPkcs1Command =
+        deserialize(cmd_data).unwrap_or_else(|e| panic!("error parsing Code::SignPss: {e:?}"));
+
+    if let Some(obj) = state
+        .objects
+        .get(command.key_id, object::Type::AsymmetricKey)
+    {
+        if let Payload::RsaKey(private_key) = &obj.payload {
+            let signature = match command.digest.len() {
+                len if len == <Sha1 as OutputSizeUser>::OutputSize::USIZE => {
+                    sign_pkcs1v15_prehash::<Sha1>(private_key, command.digest.as_ref())
+                }
+                len if len == <Sha256 as OutputSizeUser>::OutputSize::USIZE => {
+                    sign_pkcs1v15_prehash::<Sha256>(private_key, command.digest.as_ref())
+                }
+                len if len == <Sha384 as OutputSizeUser>::OutputSize::USIZE => {
+                    sign_pkcs1v15_prehash::<Sha384>(private_key, command.digest.as_ref())
+                }
+                len if len == <Sha512 as OutputSizeUser>::OutputSize::USIZE => {
+                    sign_pkcs1v15_prehash::<Sha512>(private_key, command.digest.as_ref())
+                }
+                len => {
+                    debug!("invalid digest length: {}", len);
+                    return device::ErrorKind::InvalidCommand.into();
+                }
+            };
+
+            SignPkcs1Response((&signature).into()).serialize()
+        } else {
+            debug!("not an Rsa key: {:?}", obj.algorithm());
+            device::ErrorKind::InvalidCommand.into()
+        }
+    } else {
+        debug!("no such object ID: {:?}", command.key_id);
+        device::ErrorKind::ObjectNotFound.into()
+    }
+}
+
 /// Verify the HMAC tag for the given data
 fn verify_hmac(state: &State, cmd_data: &[u8]) -> response::Message {
     let command: VerifyHmacCommand =

src/mockhsm/object/payload.rs

@@ -80,6 +80,13 @@ impl Payload {
 
     /// Generate a new key with the given algorithm
     pub fn generate(algorithm: Algorithm) -> Self {
+        fn gen_rsa(len: usize) -> Payload {
+            let private_key =
+                rsa::RsaPrivateKey::new(&mut OsRng, len).expect("failed to generate a key");
+
+            Payload::RsaKey(private_key)
+        }
+
         match algorithm {
             Algorithm::Wrap(wrap_alg) => {
                 let mut bytes = vec![0u8; wrap_alg.key_len()];
@@ -96,6 +103,9 @@ impl Payload {
                 asymmetric::Algorithm::Ed25519 => {
                     Payload::Ed25519Key(ed25519::SigningKey::generate(&mut OsRng))
                 }
+                asymmetric::Algorithm::Rsa2048 => gen_rsa(2048),
+                asymmetric::Algorithm::Rsa3072 => gen_rsa(3072),
+                asymmetric::Algorithm::Rsa4096 => gen_rsa(4096),
                 _ => {
                     panic!("MockHsm doesn't support this asymmetric algorithm: {asymmetric_alg:?}")
                 }

src/rsa.rs

@@ -1,11 +1,5 @@
 //! RSA (Rivest–Shamir–Adleman) asymmetric cryptosystem support
 //! (signing/encryption).
-//!
-//! NOTE: This functionality has not been properly tested and is therefore
-//! not enabled by default! Enable the `untested` cargo feature if you would
-//! like to use it (please report success or bugs!)
-
-// TODO(tarcieri): finalize and test RSA support
 
 mod algorithm;
 pub mod mgf;