ossl_rsa_sp800_56b_check_public: Be more lenient with small keys
Fixes
For small keys the MR test on the modulus can return
BN_PRIMETEST_COMPOSITE_WITH_FACTOR status although the modulus
is correct.
@@ -290,21 +290,19 @@ int ossl_rsa_get_lcm(BN_CTX *ctx, const BIGNUM *p, const BIGNUM *q,
int ossl_rsa_sp800_56b_check_public(const RSA *rsa)
{
int ret = 0, status;
-#ifdef FIPS_MODULE
int nbits;
-#endif
BN_CTX *ctx = NULL;
BIGNUM *gcd = NULL;
if (rsa->n == NULL || rsa->e == NULL)
return 0;
+ nbits = BN_num_bits(rsa->n);
#ifdef FIPS_MODULE
/*
* (Step a): modulus must be 2048 or 3072 (caveat from SP800-56Br1)
* NOTE: changed to allow keys >= 2048
*/
- nbits = BN_num_bits(rsa->n);
if (!ossl_rsa_sp800_56b_validate_strength(nbits, -1)) {
ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_KEY_LENGTH);
return 0;
@@ -336,7 +334,13 @@ int ossl_rsa_sp800_56b_check_public(const RSA *rsa)
}
ret = ossl_bn_miller_rabin_is_prime(rsa->n, 0, ctx, NULL, 1, &status);
+#ifdef FIPS_MODULE
if (ret != 1 || status != BN_PRIMETEST_COMPOSITE_NOT_POWER_OF_PRIME) {
+#else
+ if (ret != 1 || (status != BN_PRIMETEST_COMPOSITE_NOT_POWER_OF_PRIME
+ && (nbits >= RSA_MIN_MODULUS_BITS
+ || status != BN_PRIMETEST_COMPOSITE_WITH_FACTOR))) {
+#endif
ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_MODULUS);
ret = 0;
goto err;
